WIP: source: HDFS

.ci/Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:16.04
+
+ENV HADOOP_HOME /usr/local/hadoop
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+
+# install packages
+RUN \
+  apt-get update && apt-get install -y \
+  ssh \
+  rsync \
+  vim \
+  openjdk-8-jdk
+
+RUN \
+  wget http://apache.mirrors.tds.net/hadoop/common/hadoop-3.1.2/hadoop-3.1.2.tar.gz && \
+  tar -xzf hadoop-3.1.2.tar.gz && \
+  rm -rf hadoop-3.1.2.tar.gz && \
+  mv hadoop-3.1.2 /usr/local && \
+  ln -sf /usr/local/hadoop-3.1.2/ /usr/local/hadoop
+
+ADD configs/hadoop_config /usr/local/hadoop
+CMD cat /usr/local/hadoop/hadoop_config >> ~/.bashrc
+
+CMD bash ~/.bashrc
+
+# copy hadoop configs
+ADD configs/*xml $HADOOP_HOME/etc/hadoop/
+
+# copy script to start hadoop
+ADD start-hadoop.sh start-hadoop.sh
+
+# expose various ports
+EXPOSE 9000 8088 50070 50075 50030 50060
+
+# start hadoop
+RUN chmod a+x /start-hadoop.sh
+CMD bash start-hadoop.sh

.ci/Dockerfile-hadoop-scratch

@@ -0,0 +1,37 @@
+FROM ubuntu:16.04
+
+ENV HADOOP_HOME /usr/local/hadoop
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+
+# install packages
+RUN \
+  apt-get update && apt-get install -y \
+  ssh \
+  rsync \
+  vim \
+  openjdk-8-jdk
+
+RUN \
+  wget http://apache.mirrors.tds.net/hadoop/common/hadoop-3.1.2/hadoop-3.1.2.tar.gz && \
+  tar -xzf hadoop-3.1.2.tar.gz && \
+  rm -rf hadoop-3.1.2.tar.gz && \
+  mv hadoop-3.1.2 /usr/local && \
+  ln -sf /usr/local/hadoop-3.1.2/ /usr/local/hadoop
+
+ADD configs/hadoop_config /usr/local/hadoop
+CMD cat /usr/local/hadoop/hadoop_config >> ~/.bashrc
+
+CMD bash ~/.bashrc
+
+# copy hadoop configs
+ADD configs/*xml $HADOOP_HOME/etc/hadoop/
+
+# copy script to start hadoop
+ADD start-hadoop.sh start-hadoop.sh
+
+# expose various ports
+EXPOSE 9000 8088 50070 50075 50030 50060
+
+# start hadoop
+RUN chmod a+x /start-hadoop.sh
+CMD bash start-hadoop.sh

.ci/Dockerfile-maven

@@ -0,0 +1,29 @@
+# Usage:
+# $ docker build -t dffml:hadoop - < .ci/Dockerfile-maven
+# $ docker run --rm -ti --net=host dffml:hadoop
+FROM maven:latest
+
+RUN \
+  apt-get update && apt-get install -y \
+  ssh \
+  rsync \
+  wget \
+  vim \
+  unzip 
+
+RUN \
+  cd /usr/local && \
+  curl -L http://apache.mirrors.tds.net/hadoop/common/hadoop-2.9.2/hadoop-2.9.2.tar.gz | tar -xz && \
+  ln -sf /usr/local/hadoop-2.9.2/ /usr/local/hadoop
+
+WORKDIR /usr/local/hadoop
+
+# From: https://stackoverflow.com/questions/51118358/noclassdeffounderror-org-apache-hadoop-yarn-server-timelineservice-collector-tim
+RUN cp ./share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-2.9.2.jar ./share/hadoop/yarn/ && \
+	mkdir -p target/test/data/dfs/name1 && \
+	mkdir -p target/test/data/dfs/name2
+
+ENV HADOOP_CLASSPATH share/hadoop/yarn/test/hadoop-yarn-server-tests-2.9.2-tests.jar
+
+CMD ["bin/hadoop", "jar", "./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.9.2-tests.jar", "minicluster", "-rmport", "50070", "-jhsport", "50071", "-format"]
+

.ci/configs/core-site.xml

@@ -0,0 +1,14 @@
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+    </property>
+    <property>
+        <name>hadoop.proxyuser.hadoopgroup.groups</name>
+        <value>*</value>
+    </property>
+    <property>
+        <name>hadoop.proxyuser.hadoopuser.hosts</name>
+        <value>*</value>
+    </property>
+</configuration>

.ci/configs/hadoop_config

@@ -0,0 +1,15 @@
+# Hadoop config
+export HADOOP_PREFIX=/usr/local/hadoop
+export HADOOP_HOME=/usr/local/hadoop
+export HADOOP_MAPRED_HOME=${HADOOP_HOME}
+export HADOOP_COMMON_HOME=${HADOOP_HOME}
+export HADOOP_HDFS_HOME=${HADOOP_HOME}
+export YARN_HOME=${HADOOP_HOME}
+export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
+# Native path
+export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_PREFIX}/lib/native
+export HADOOP_OPTS="-Djava.library.path=$HADOOP_PREFIX/lib/native"
+# Java path
+export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
+# OS path
+export PATH=$PATH:$HADOOP_HOME/bin:$JAVA_PATH/bin:$HADOOP_HOME/sbin

.ci/configs/hdfs-site.xml

@@ -0,0 +1,14 @@
+<configuration>
+    <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+    </property>
+    <property>
+        <name>dfs.name.dir</name>
+        <value>file:/usr/local/hadoop/hadoopdata/hdfs/namenode</value>
+    </property>
+    <property>
+        <name>dfs.data.dir</name>
+        <value>file:/usr/local/hadoop/hadoopdata/hdfs/datanode</value>
+    </property>
+</configuration>

.ci/configs/mapred-site.xml

@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>mapreduce.framework.name</name>
+        <value>yarn</value>
+    </property>
+</configuration>

.ci/configs/yarn-site.xml

@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+    </property>
+</configuration>

.ci/start-hadoop.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+source /usr/local/hadoop/hadoop_config
+
+# start ssh server
+/etc/init.d/ssh start
+
+# format namenode
+$HADOOP_HOME/bin/hdfs namenode -format
+
+# start hadoop
+$HADOOP_HOME/sbin/start-dfs.sh
+$HADOOP_HOME/sbin/start-yarn.sh
+#$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver
+
+# keep container running
+##tail -f /dev/null

.travis.yml

@@ -24,6 +24,7 @@ env:
   - PLUGIN=source/mysql
   - PLUGIN=feature/git
   - PLUGIN=feature/auth
+  - PLUGIN=source/hdfs
   - PLUGIN=service/http
   - CHANGELOG=1
   - WHITESPACE=1

CHANGELOG.md

@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Mailing list info
 - Issue template for questions
 - Multiple Scikit Models with dynamic config
+- To work with data from a Hadoop source
 - Entrypoint listing command to development service to aid in debugging issues
   with entrypoints.
 - HTTP API service to enable interacting with DFFML over HTTP. Currently

source/hdfs/.coveragerc

@@ -0,0 +1,13 @@
+[run]
+source =
+    dffml_source_hdfs
+    tests
+branch = True
+
+[report]
+exclude_lines =
+    no cov
+    no qa
+    noqa
+    pragma: no cover
+    if __name__ == .__main__.:

source/hdfs/.gitignore

@@ -0,0 +1,20 @@
+*.log
+*.pyc
+.cache/
+.coverage
+.idea/
+.vscode/
+*.egg-info/
+build/
+dist/
+docs/build/
+venv/
+wheelhouse/
+*.egss
+.mypy_cache/
+*.swp
+.venv/
+.eggs/
+*.modeldir
+*.db
+htmlcov/

source/hdfs/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2019 Intel
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

source/hdfs/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include LICENSE

source/hdfs/README.md

@@ -0,0 +1,16 @@
+# DFFML REPLACE_PACKGE_NAME Service
+
+## About
+
+Description
+
+## Usage
+
+```console
+# Command line usage here
+```
+
+## License
+
+REPLACE_PACKGE_NAME Service is distributed under the terms of the
+[MIT License](LICENSE).

source/hdfs/dffml_setup_common.py

@@ -0,0 +1,79 @@
+import os
+import sys
+import ast
+from io import open
+from pathlib import Path
+from setuptools import find_packages
+
+ORG = "pdxjohnny"
+NAME = "dffml-source-hdfs"
+DESCRIPTION = "DFFML Source for HDFS"
+AUTHOR_NAME = "Sudharsana K J L"
+AUTHOR_EMAIL = "[email protected]"
+# Install dffml if it is not installed in development mode
+INSTALL_REQUIRES = [] + (
+    ["dffml>=0.2.1"]
+    if not any(
+        list(
+            map(
+                os.path.isfile,
+                list(
+                    map(
+                        lambda syspath: os.path.join(
+                            syspath, "dffml.egg-link"
+                        ),
+                        sys.path,
+                    )
+                ),
+            )
+        )
+    )
+    else []
+)
+
+IMPORT_NAME = (
+    NAME
+    if "replace_package_name".upper() != NAME
+    else "replace_import_package_name".upper()
+).replace("-", "_")
+
+SELF_PATH = Path(sys.argv[0]).parent.resolve()
+if not (SELF_PATH / Path(IMPORT_NAME, "version.py")).is_file():
+    SELF_PATH = os.path.dirname(os.path.realpath(__file__))
+
+VERSION = ast.literal_eval(
+    Path(SELF_PATH, IMPORT_NAME, "version.py")
+    .read_text()
+    .split("=")[-1]
+    .strip()
+)
+
+README = Path(SELF_PATH, "README.md").read_text()
+
+SETUP_KWARGS = dict(
+    name=NAME,
+    version=VERSION,
+    description=DESCRIPTION,
+    long_description=README,
+    long_description_content_type="text/markdown",
+    author=AUTHOR_NAME,
+    author_email=AUTHOR_EMAIL,
+    maintainer="John Andersen",
+    maintainer_email="[email protected]",
+    url=f"https://github.com/{ORG}/{NAME}",
+    license="MIT",
+    keywords=["dffml"],
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Natural Language :: English",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Programming Language :: Python :: Implementation :: PyPy",
+    ],
+    install_requires=INSTALL_REQUIRES,
+    packages=find_packages(),
+)