-
Notifications
You must be signed in to change notification settings - Fork 1
/
DistributedFile.sh
124 lines (93 loc) · 6.19 KB
/
DistributedFile.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# Generate java home and hadoop home
JAVA_HOME=$cd %java_home%
echo "$JAVA_HOME"
HADOOP_HOME=$cd %hadoop_home%
echo "$HADOOP_HOME"
WIN_UTILS=$winutils chmod -R 777 C:\hadoop-2.8.0
echo "$WIN_UTILS"
# Call datanode and namenode on hadoop
NODE=$cd sbin\start-dfs.cmd
echo "$NODE"
HADOOP_FORMAT=$hadoop namenode -format
echo "$HADOOP_FORMAT"
# Check hadoop directory file system
DIR_FILE=$hadoop fs -ls hdfs://localhost:9000/
echo "$DIR_FILE"
# Create directory project
DIR_PROJECT=$hadoop fs -mkdir hdfs://localhost:9000/spotify
echo "$DIR_PROJECT"
# Upload data from local to hadoop directory file system
DATA=$hadoop fs -copyFromLocal D:\spotify\spotify_millsongdata.csv hdfs://localhost:9000/spotify/spotify-lyrics.csv
echo "$DATA"
# Check data in hadoop directory file system
DIR_FILE=$hadoop fs -ls hdfs://localhost:9000/spotify/
echo "$DIR_FILE"
# Read lazada items dataset in hadoop file system
DIR_FILE=$hadoop fs -cat hdfs://localhost:9000/spotify/spotify-lyrics.csv
echo "$DIR_FILE"
# Rename hadoop file system of analysis abba lyrics result from hive system
FILE_RENAME=$hadoop fs -mv hdfs://localhost:9000/spotify/abba/000000_0 hdfs://localhost:9000/spotify/abba/lyrics.txt
echo "$FILE_RENAME"
# Rename hadoop file system of analysis alabama lyrics result from hive system
FILE_RENAME=$hadoop fs -mv hdfs://localhost:9000/spotify/alabama/000000_0 hdfs://localhost:9000/spotify/alabama/lyrics.txt
echo "$FILE_RENAME"
# Rename hadoop file system of analysis bob dylan lyrics result from hive system
FILE_RENAME=$hadoop fs -mv hdfs://localhost:9000/spotify/bob-dylan/000000_0 hdfs://localhost:9000/spotify/bob-dylan/lyrics.txt
echo "$FILE_RENAME"
# Rename hadoop file system of analysis donna summer lyrics result from hive system
FILE_RENAME=$hadoop fs -mv hdfs://localhost:9000/spotify/donna-summer/000000_0 hdfs://localhost:9000/spotify/donna-summer/lyrics.txt
echo "$FILE_RENAME"
# Generate Mapreduce on abba lyrics in hadoop system
MAP_RED=$hadoop jar c:\hadoop-2.8.0\share\hadoop\tools\lib\hadoop-streaming-*.jar -file "D:\mapper.py" -mapper "python D:\mapper.py" -file "D:\reducer.py" -reducer "python D:\reducer.py" -input hdfs://localhost:9000/spotify/abba/lyrics.txt -output hdfs://localhost:9000/spotify/abba/output/
echo "$MAP_RED"
# Generate Mapreduce on alabama lyrics in hadoop system
MAP_RED=$hadoop jar c:\hadoop-2.8.0\share\hadoop\tools\lib\hadoop-streaming-*.jar -file "D:\mapper.py" -mapper "python D:\mapper.py" -file "D:\reducer.py" -reducer "python D:\reducer.py" -input hdfs://localhost:9000/spotify/alabama/lyrics.txt -output hdfs://localhost:9000/spotify/alabama/output/
echo "$MAP_RED"
# Generate Mapreduce on bob dylan lyrics in hadoop system
MAP_RED=$hadoop jar c:\hadoop-2.8.0\share\hadoop\tools\lib\hadoop-streaming-*.jar -file "D:\mapper.py" -mapper "python D:\mapper.py" -file "D:\reducer.py" -reducer "python D:\reducer.py" -input hdfs://localhost:9000/spotify/bob-dylan/lyrics.txt -output hdfs://localhost:9000/spotify/bob-dylan/output/
echo "$MAP_RED"
# Generate Mapreduce on donna summer lyrics in hadoop system
MAP_RED=$hadoop jar c:\hadoop-2.8.0\share\hadoop\tools\lib\hadoop-streaming-*.jar -file "D:\mapper.py" -mapper "python D:\mapper.py" -file "D:\reducer.py" -reducer "python D:\reducer.py" -input hdfs://localhost:9000/spotify/donna-summer/lyrics.txt -output hdfs://localhost:9000/spotify/donna-summer/output/
echo "$MAP_RED"
# Generate Mapreduce on george strait lyrics in hadoop system
MAP_RED=$hadoop jar c:\hadoop-2.8.0\share\hadoop\tools\lib\hadoop-streaming-*.jar -file "D:\mapper.py" -mapper "python D:\mapper.py" -file "D:\reducer.py" -reducer "python D:\reducer.py" -input hdfs://localhost:9000/spotify/george-strait/lyrics.txt -output hdfs://localhost:9000/spotify/george-strait/output/
echo "$MAP_RED"
# Generate Mapreduce on gordon lightfoot lyrics in hadoop system
MAP_RED=$hadoop jar c:\hadoop-2.8.0\share\hadoop\tools\lib\hadoop-streaming-*.jar -file "D:\mapper.py" -mapper "python D:\mapper.py" -file "D:\reducer.py" -reducer "python D:\reducer.py" -input hdfs://localhost:9000/spotify/gordon-lightfoot/lyrics.txt -output hdfs://localhost:9000/spotify/gordon-lightfoot/output/
echo "$MAP_RED"
# Check mapreduce result on abba directory
FILE_RESULT=$hadoop fs -cat hdfs://localhost:9000/spotify/abba/output/part-00000
echo "$FILE_RESULT"
# Check mapreduce result on alabama directory
FILE_RESULT=$hadoop fs -cat hdfs://localhost:9000/spotify/alabama/output/part-00000
echo "$FILE_RESULT"
# Check mapreduce result on donna summer directory
FILE_RESULT=$hadoop fs -cat hdfs://localhost:9000/spotify/donna-summer/output/part-00000
echo "$FILE_RESULT"
# Check mapreduce result on bob dylan directory
FILE_RESULT=$hadoop fs -cat hdfs://localhost:9000/spotify/bob-dylan/output/part-00000
echo "$FILE_RESULT"
# Check mapreduce result on george strait directory
FILE_RESULT=$hadoop fs -cat hdfs://localhost:9000/spotify/george-strait/output/part-00000
echo "$FILE_RESULT"
# Check mapreduce result on gordon lightfoot directory
FILE_RESULT=$hadoop fs -cat hdfs://localhost:9000/spotify/gordon-lightfoot/output/part-00000
echo "$FILE_RESULT"
# Download abba mapreduce result in local computer
OUT_FILE=$hadoop fs -copyToLocal hdfs://localhost:9000/spotify/abba/output/part-00000 D:\spotify\abba-lyrics.txt
echo "$OUT_FILE"
# Download alabama mapreduce result in local computer
OUT_FILE=$hadoop fs -copyToLocal hdfs://localhost:9000/spotify/alabama/output/part-00000 D:\spotify\alabama-lyrics.txt
echo "$OUT_FILE"
# Download donna summer mapreduce result in local computer
OUT_FILE=$hadoop fs -copyToLocal hdfs://localhost:9000/spotify/donna-summer/output/part-00000 D:\spotify\donna-summer-lyrics.txt
echo "$OUT_FILE"
# Download bob dylan mapreduce result in local computer
OUT_FILE=$hadoop fs -copyToLocal hdfs://localhost:9000/spotify/bob-dylan/output/part-00000 D:\spotify\bob-dylan-lyrics.txt
echo "$OUT_FILE"
# Download george strait mapreduce result in local computer
OUT_FILE=$hadoop fs -copyToLocal hdfs://localhost:9000/spotify/george-strait/output/part-00000 D:\spotify\george-strait-lyrics.txt
echo "$OUT_FILE"
# Download gordon lightfoot mapreduce result in local computer
OUT_FILE=$hadoop fs -copyToLocal hdfs://localhost:9000/spotify/gordon-lightfoot/output/part-00000 D:\spotify\gordon-lightfoot-lyrics.txt
echo "$OUT_FILE"