Wednesday, June 10, 2015

HDFS COMMANDS

12:14 PM - By ajay desai 0


  1) -ls /: - This command is used to display the list of directories and files in the hadoop file system.

     root@ubuntu:/home# hadoop fs -ls /
     Found 7 items
     drwxr-xr-x   - root   supergroup          0 2013-05-08 21:28 /HdfsBatch11
     drwxr-xr-x   - root   supergroup          0 2012-12-18 07:34 /etc
     drwxr-xr-x   - root   supergroup          0 2013-01-24 08:51 /hbase
     -rw-r--r--   1 root   supergroup         56 2013-05-21 03:21 /piglab
     drwxr-xr-x   - root   supergroup          0 2013-05-21 04:45 /tmp
     drwxr-xr-x   - root   supergroup          0 2015-04-30 06:34 /user
     drwxr-xr-x   - mapred supergroup          0 2015-05-05 03:22 /var

  2) hadoop dfsadmin -safemode get: - this command is used to get the status of hadoop name node.
     root@ubuntu:/home# hadoop dfsadmin -safemode get
     Safe mode is OFF

  3) hadoop fs -mkdir /directoryname: - this command is used to create a directory in the hadoop file system.
     root@ubuntu:/home# hadoop fs -mkdir /test49

  4) -put: - used to copy a file from local file system (LFS) to Hadoop File System (HDFS).
      root@ubuntu:/home# mkdir BATCH49
      root@ubuntu:/home# cp ajay.txt /home/BATCH49

      root@ubuntu:/home/BATCH49# ll
      total 12
      drwxr-xr-x  2 root root 4096 2015-05-05 22:05 ./
      drwxr-xr-x 15 root root 4096 2015-05-05 21:13 ../
      -rw-r--r--  1 root root  118 2015-05-05 22:05 ajay.txt

      root@ubuntu:/home/BATCH49# hadoop fs -put ajay.txt /test49
      root@ubuntu:/home/BATCH49# hadoop fs -ls /test49
      Found 1 items
      -rw-r--r--   1 root supergroup        118 2015-05-05 22:18 /test49/ajay.txt

  5) copyFromLocal: - It is same as put command which copies a file or directory from LFS to HDFS.

     root@ubuntu:/home/BATCH49# nano akshata.txt
     root@ubuntu:/home/BATCH49# ll
     total 16
     drwxr-xr-x  2 root root 4096 2015-05-06 06:54 ./
     drwxr-xr-x 15 root root 4096 2015-05-05 21:13 ../
     -rw-r--r--  1 root root  118 2015-05-05 22:05 ajay.txt
     -rw-r--r--  1 root root  103 2015-05-06 06:54 akshata.txt

     root@ubuntu:/home/BATCH49# hadoop fs -copyFromLocal akshata.txt /test49
     root@ubuntu:/home/BATCH49# hadoop fs -ls /test49
     Found 2 items
     -rw-r--r--   1 root supergroup        118 2015-05-05 22:18 /test49/ajay.txt
     -rw-r--r--   1 root supergroup        103 2015-05-06 06:55 /test49/akshata.txt

      root@ubuntu:/home/BATCH49# nano vijay.log
      root@ubuntu:/home/BATCH49# nano abhay.log
      root@ubuntu:/home/BATCH49# ll
      total 24
      drwxr-xr-x  2 root root 4096 2015-05-06 07:48 ./
      drwxr-xr-x 15 root root 4096 2015-05-05 21:13 ../
      -rw-r--r--  1 root root   57 2015-05-06 07:48 abhay.log
      -rw-r--r--  1 root root  118 2015-05-06 07:18 ajay.txt
      -rw-r--r--  1 root root  103 2015-05-06 07:11 akshata.txt
      -rw-r--r--  1 root root   59 2015-05-06 07:48 vijay.log

      root@ubuntu:/home/BATCH49# hadoop fs -put *.log /test49 (hadoop fs -copyFromLocal *.log /test49)
 
      root@ubuntu:/home/BATCH49# hadoop fs -ls /test49
      Found 4 items
      -rw-r--r--   1 root supergroup         57 2015-05-06 08:21 /test49/abhay.log
      -rw-r--r--   1 root supergroup        118 2015-05-05 22:18 /test49/ajay.txt
      -rw-r--r--   1 root supergroup        103 2015-05-06 06:55 /test49/akshata.txt
      -rw-r--r--   1 root supergroup         59 2015-05-06 08:21 /test49/vijay.log

  6) get: - it is used to copy a file or directory from HDFS to LFS.
   
     root@ubuntu:/home/BATCH49# hadoop fs -get /test49/akshata.txt /home/BATCH49
     get: Target /home/BATCH49/akshata.txt already exists

     root@ubuntu:/home/BATCH49# rm akshata.txt
     root@ubuntu:/home/BATCH49# hadoop fs -get /test49/akshata.txt /home/BATCH49

  7) copyToLocal: - it is same as get command which copies a file or directory from HDFS to LFS.

     root@ubuntu:/home/BATCH49# hadoop fs -copyToLocal /test49/ajay.txt /home/BATCH49
     copyToLocal: Target /home/BATCH49/ajay.txt already exists

     root@ubuntu:/home/BATCH49# rm ajay.txt
     root@ubuntu:/home/BATCH49# hadoop fs -copyToLocal /test49/ajay.txt /home/BATCH49

  8) cat: - to view the contents of a file present in a directory.

     root@ubuntu:/home/BATCH49# hadoop fs -cat /test49/ajay.txt
     Hi I am Ajay, I aspire to work as a JAVA developer in Yahoo.Inc..
     My friend Ganesh works as Hadoop analyst in Google.

  9) cp: - to copy the content of one HDFS directory into another HDFS directory.

     root@ubuntu:/home/BATCH49# hadoop fs -mkdir /test49n
     root@ubuntu:/home/BATCH49# hadoop fs -ls /test49n

     root@ubuntu:/home/BATCH49# hadoop fs -cp /test49/*.log /test49n
     root@ubuntu:/home/BATCH49# hadoop fs -ls /test49n
     Found 2 items
     -rw-r--r--   1 root supergroup         57 2015-05-06 09:15 /test49n/abhay.log
     -rw-r--r--   1 root supergroup         59 2015-05-06 09:15 /test49n/vijay.log

 10) touchz: - to create an empty file in a HDFS directory.

     root@ubuntu:/home/BATCH49# hadoop fs -touchz /test49n/uma.log
     root@ubuntu:/home/BATCH49# hadoop fs -ls /test49n
     Found 3 items
     -rw-r--r--   1 root supergroup         57 2015-05-06 09:15 /test49n/abhay.log
     -rw-r--r--   1 root supergroup          0 2015-05-06 10:26 /test49n/uma.log
     -rw-r--r--   1 root supergroup         59 2015-05-06 09:15 /test49n/vijay.log
     root@ubuntu:/home/BATCH49# hadoop fs -cat /test49n/uma.log
     root@ubuntu:/home/BATCH49#

 11) wc: - To get the no: of words, characters and lines in a file present in a HDFS directory.
     root@ubuntu:/home# hadoop fs -ls /test49/abhay.log|wc
      2      11      91

 12) wc -l: - To get the no: of lines in a file present in a HDFS directory.
     root@ubuntu:/home# hadoop fs -ls /test49/abhay.log|wc -l
     2

 13) wc -w: - To get the no: of words in a file present in a HDFS directory.
     root@ubuntu:/home# hadoop fs -ls /test49/abhay.log|wc -w
     11

 14) wc -c: - To get the no: of characters in a file present in a HDFS directory.
     root@ubuntu:/home# hadoop fs -ls /test49/abhay.log|wc -c
     91

 15) -count: - This command is used to count the no: of files and directories in the present working directory or within HDFS. It is also used to give a summary of

     bytes occupied by all the files and directories in the present working directory or within HDFS.

     root@ubuntu:/home# hadoop fs -count /test49
                1            4                337 hdfs://localhost:8020/test49
   
     Here, '1' represents directory, '4' represents no: of files and '337' represents no: of bytes occupied by the directory in the HDFS.


 16) setrep: - This command is used to set the replication factor for a particular file or directory present in HDFS.
     root@ubuntu:/home# hadoop fs -setrep 3 -w /test49n
     Replication 3 set: hdfs://localhost:8020/test49n/abhay.log
     Replication 3 set: hdfs://localhost:8020/test49n/uma.log
     Replication 3 set: hdfs://localhost:8020/test49n/vijay.log
     Waiting for hdfs://localhost:8020/test49n/abhay.log.....^Croot@ubuntu:/home#

 17) ctrl+c: - It is used to kill the current process.

 18) fsck: - This command is used to give information about the no: of physical blocks allocated for a file or directory and no: of replications for a particular file

     or directory.

     root@ubuntu:/home# hadoop fsck -blocks /test49n
FSCK started by root (auth:SIMPLE) from /192.168.126.132 for path /test49n at Thu May 07 05:40:06 PDT 2015
.
/test49n/abhay.log:  Under replicated blk_5592892966046075189_3918. Target Replicas is 3 but found 1 replica(s).
..
/test49n/vijay.log:  Under replicated blk_-3398363587046948672_3919. Target Replicas is 3 but found 1 replica(s).
Status: HEALTHY
 Total size: 116 B
 Total dirs: 1
 Total files: 3
 Total blocks (validated): 2 (avg. block size 58 B)
 Minimally replicated blocks: 2 (100.0 %)
 Over-replicated blocks: 0 (0.0 %)
 Under-replicated blocks: 2 (100.0 %)
 Mis-replicated blocks: 0 (0.0 %)
 Default replication factor: 1
 Average block replication: 1.0
 Corrupt blocks: 0
 Missing replicas: 4 (200.0 %)
 Number of data-nodes: 1
 Number of racks: 1
FSCK ended at Thu May 07 05:40:06 PDT 2015 in 1 milliseconds


The filesystem under path '/test49n' is HEALTHY


 19) mv: - this command is used to move a file or group of files from one HDFS directory to another HDFS directory.
      root@ubuntu:/home# hadoop fs -mv /test49/ajay.txt /test49n  
 
 20) rm: - This command is used to remove a file from a HDFS directory.
     root@ubuntu:/home# hadoop fs -rm /test49n/ajay.txt
     Deleted hdfs://localhost:8020/test49n/ajay.txt

 21)  rmr: - to delete a directory from HDFS
      root@ubuntu:/home/BATCH49# hadoop fs -rmr /test49n
      Deleted hdfs://localhost:8020/test49n
 

 22) lsr: - to show the list of all files, directories, subdirectories and files within the directories in HDFS.
   
     root@ubuntu:/home/BATCH49# hadoop fs -mkdir /test49
     root@ubuntu:/home/BATCH49# hadoop fs -mkdir /test49/test49n

     root@ubuntu:/home/BATCH49# hadoop fs -ls /test49
     Found 1 items
     drwxr-xr-x   - root supergroup          0 2015-05-07 09:44 /test49/test49n
   
     root@ubuntu:/home/BATCH49# ll
     total 24
     drwxr-xr-x  2 root root 4096 2015-05-07 09:45 ./
     drwxr-xr-x 15 root root 4096 2015-05-05 21:13 ../
     -rw-r--r--  1 root root   57 2015-05-06 07:48 abhay.log
     -rw-r--r--  1 root root  118 2015-05-06 07:18 ajay.txt
     -rw-r--r--  1 root root  103 2015-05-06 07:11 akshata.txt
     -rw-r--r--  1 root root   59 2015-05-06 07:48 vijay.log

     root@ubuntu:/home/BATCH49# hadoop fs -put ajay.txt /test49
     root@ubuntu:/home/BATCH49# hadoop fs -cp /test49/ajay.txt /test49/test49n
     root@ubuntu:/home/BATCH49# hadoop fs -lsr /test49
     -rw-r--r--   1 root supergroup        118 2015-05-07 09:45 /test49/ajay.txt
     drwxr-xr-x   - root supergroup          0 2015-05-07 09:46 /test49/test49n
     -rw-r--r--   1 root supergroup        118 2015-05-07 09:46 /test49/test49n/ajay.txt

 23) chown: - to change the owner of a file or directory in HDFS.
     root@ubuntu:/home/BATCH49# hadoop fs -chown mrinmoy /test49
     chown: changing ownership of 'hdfs://localhost:8020/test49':org.apache.hadoop.security.AccessControlException: Non-super user cannot change owner.
 

 24) hadoop dfsadmin -report: - It is used to generate a report about the memory capacity used by HDFS and also about the available datanodes.
   
   root@ubuntu:/home# hadoop dfsadmin -report
   Configured Capacity: 30352158720 (28.27 GB)
   Present Capacity: 25611059200 (23.85 GB)
   DFS Remaining: 25510649856 (23.76 GB)
   DFS Used: 100409344 (95.76 MB)
   DFS Used%: 0.39%
   Under replicated blocks: 289
   Blocks with corrupt replicas: 0
   Missing blocks: 0

  -------------------------------------------------
   Datanodes available: 1 (1 total, 0 dead)
   Name: 127.0.0.1:50010
   Decommission Status : Normal
   Configured Capacity: 30352158720 (28.27 GB)
   DFS Used: 100409344 (95.76 MB)
   Non DFS Used: 4741099520 (4.42 GB)
   DFS Remaining: 25510649856(23.76 GB)
   DFS Used%: 0.33%
   DFS Remaining%: 84.05%
   Last contact: Fri May 08 20:06:37 PDT 2015


 25) hadoop dfsadmin -safemode ENTER: - To change the safe mode status of HADOOP name node to ON. When the safe mode status is ON, no files or directories cannot be
     copied from LFS to HDFS. we cannot copy or move files or directories within HDFS and we cannot delete any file or directory in HDFS.

     root@ubuntu:/home/BATCH49# hadoop dfsadmin -safemode ENTER
     Safe mode is ON
     root@ubuntu:/home/BATCH49# hadoop fs -copyFromLocal ajay.txt /test49
     copyFromLocal: org.apache.hadoop.hdfs.server.namenode.SafeModeException: Cannot create file/test49/ajay.txt. Name node is in safe mode.

     root@ubuntu:/home/BATCH49# hadoop dfsadmin -safemode LEAVE
     Safe mode is OFF
     root@ubuntu:/home/BATCH49# hadoop fs -put ajay.txt /test49

     root@ubuntu:/home/BATCH49# hadoop dfsadmin -safemode ENTER
     Safe mode is ON
     root@ubuntu:/home/BATCH49# hadoop fs -cp /test49/ajay.txt /test49/test49n
     cp: org.apache.hadoop.hdfs.server.namenode.SafeModeException: Cannot create file/test49/test49n/ajay.txt. Name node is in safe mode.

     root@ubuntu:/home/BATCH49# hadoop fs -mv /test49/ajay.txt /test49/test49n
     mv: org.apache.hadoop.hdfs.server.namenode.SafeModeException: Cannot rename /test49/ajay.txt. Name node is in safe mode.
   
     root@ubuntu:/home/BATCH49# hadoop fs -rm /test49/ajay.txt
     rm: org.apache.hadoop.hdfs.server.namenode.SafeModeException: Cannot delete /test49/ajay.txt. Name node is in safe mode.
 
 26) hadoop dfsadmin -safemode LEAVE: - To change the safe mode status of HADOOP name node to OFF, so that all copy, move and delete operations can be performed.

     root@ubuntu:/home/BATCH49# hadoop dfsadmin -safemode LEAVE
     Safe mode is OFF
     root@ubuntu:/home/BATCH49# hadoop fs -put ajay.txt /test49
       


Tags:
About the Author

I am Azeheruddin Khan having more than 6 year experience in c#, Asp.net and ms sql.My work comprise of medium and enterprise level projects using asp.net and other Microsoft .net technologies. Please feel free to contact me for any queries via posting comments on my blog,i will try to reply as early as possible. Follow me @fresher2programmer
View all posts by admin →

Get Updates

Subscribe to our e-mail newsletter to receive updates.

Share This Post

0 comments:

adsense

© 2014 Fresher2Programmer. WP Theme-junkie converted by Bloggertheme9
Powered by Blogger.
back to top