distributed grep using MapReduce

.idea/.gitignore

+# Default ignored files
+/shelf/
+/workspace.xml

.idea/compiler.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <annotationProcessing>
+      <profile name="Maven default annotation processors profile" enabled="true">
+        <sourceOutputDir name="target/generated-sources/annotations" />
+        <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
+        <outputRelativeToContentRoot value="true" />
+        <module name="DistributedGrep" />
+      </profile>
+    </annotationProcessing>
+  </component>
+</project>
\ No newline at end of file

.idea/encodings.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding">
+    <file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
+    <file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
+  </component>
+</project>
\ No newline at end of file

.idea/jarRepositories.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RemoteRepositoriesConfiguration">
+    <remote-repository>
+      <option name="id" value="central" />
+      <option name="name" value="Central Repository" />
+      <option name="url" value="https://repo.maven.apache.org/maven2" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="central" />
+      <option name="name" value="Maven Central repository" />
+      <option name="url" value="https://repo1.maven.org/maven2" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="jboss.community" />
+      <option name="name" value="JBoss Community repository" />
+      <option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
+    </remote-repository>
+  </component>
+</project>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ExternalStorageConfigurationManager" enabled="true" />
+  <component name="MavenProjectsManager">
+    <option name="originalFiles">
+      <list>
+        <option value="$PROJECT_DIR$/pom.xml" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>
\ No newline at end of file

Read Me.txt

+-run hadoop first
+
+- to compile classes and generate jar with needed outer dependencies:
+ use:   mvn clean package
+
+- take the resulted jar (has shaded or does not have original) , rename it to : distr-grep.jar.
+
+use it as :
+hadoop jar distr-grep.jar org.example.DistributedGrep %HDFS_LOC%/input %HDFS_LOC%/output "pattern"

pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>DistributedGrep</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>2.6.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-core</artifactId>
+            <version>1.2.1</version>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.4</version>
+                <configuration>
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
+                    <transformers>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            <manifestEntries>
+                                <Main-Class>org.example.DistributedGrep</Main-Class>
+                                <Class-Path>${hadoop_cp}</Class-Path>
+                            </manifestEntries>
+                        </transformer>
+                    </transformers>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file

src/main/java/org/example/DistributedGrep.java

+package org.example;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+import java.io.IOException;
+
+public class DistributedGrep
+{
+
+    public static class GrepMapper extends Mapper<LongWritable, Text, Text, Text>
+    {
+        private Text outputKey = new Text();
+        private Text outputValue = new Text();
+        private String pattern;
+
+        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException
+        {
+            if (args.length != 3)
+            {
+                System.err.println("Usage: DistributedGrep.jar <input_path> <output_path> <pattern>");
+                System.exit(1);
+            }
+
+            String inputPath = args[0];
+            String outputPath = args[1];
+            String pattern = args[2];
+
+            Configuration conf = new Configuration();
+
+            Job job = Job.getInstance(conf, "Distributed-Grep");
+            job.setJarByClass(DistributedGrep.class);
+
+            job.setMapperClass(GrepMapper.class);
+            job.setReducerClass(GrepReducer.class);
+
+            job.setOutputKeyClass(Text.class);
+            job.setOutputValueClass(Text.class);
+
+            FileInputFormat.addInputPath(job, new Path(inputPath));
+            FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+
+            System.exit(job.waitForCompletion(true) ? 0 : 1);
+        }
+
+
+        @Override
+        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException
+        {
+            String line = value.toString();
+            if (line.contains(pattern))
+            {
+                outputKey.set(line);
+                outputValue.set("");
+                context.write(outputKey, outputValue);
+            }
+        }
+    }
+
+    public static class GrepReducer extends Reducer<Text, Text, Text, Text>
+    {
+        @Override
+        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+            context.write(key, new Text());
+        }
+    }
+
+}
\ No newline at end of file

target/maven-archiver/pom.properties

+artifactId=DistributedGrep
+groupId=org.example
+version=1.0-SNAPSHOT

target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst

+org\example\DistributedGrep$GrepReducer.class
+org\example\DistributedGrep.class
+org\example\DistributedGrep$GrepMapper.class

target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst

+D:\HIAST\FIY\SS\Cloud Computing\Lab\3\DistributedGrep\src\main\java\org\example\DistributedGrep.java