Bash pills

To read a file in a loop:

To read the file line by line use:

   cat sample.txt | while read line; do echo $line; done

To read the file element (word) by element you can use:

   for word in `cat sample.txt`; do echo $word; done

To print the LAST field using awk use $NF

awk '{print $NF}'

This is useful if you don't know how many fields you will have, so you can't use field numbers 

# remove duplicate lines / unique lines

uniq filename 

#Compute RMSE from two files (GT.txt and Predicted.txt) in one line.

 paste <(cat GT.txt) <(cat Predicted.txt) | awk '{accum = accum + (($1-$2)*($1-$2))} {n=n+1} END {print "MRSE=" sqrt(accum/n)}'

Inside the loop we accumulate the sum of the squared difference, also we count the number of elements (n), finally outside the loop we compute the sqrt.

#Use paste command with pipes instead of input files.

The script:

cat  test.scrle | threshold.pl 5.5 > test.bin

paste -d " " test.bin test.jpgl  | grep pos  | awk '{print $2}'

Can be written without the auxiliary file test.bin:

paste <(test.scrle | threshold.pl 5.5) <(cat test.jpgl) | grep pos | cut -f2


#Displaying an specific field/column:

Instead of using awk '{print $2}' we can use the cut command:

cut -f2

to display specific fields write them separated by commas:  -f2,6,9
to display a range of fields use colon:   -f2:6
to display all fields starting by the third one use - at the end:    -f3-



# replace all spaces into filename with underscore

find . -depth -name '* *' \
| while IFS= read -r f ; do mv -i "$f" "$(dirname "$f")/$(basename "$f"|tr ' ' _)" ; done

# lines count

BE CAREFUL wc counts the \n ( newlines) 

if the file does not have one at the end of the last line the number is WRONG

# increment variable

count=1; 

for f in $(cat visual/features_ml/lists_mlfu300/class.txt); do \

echo "$((count++))";\

 done


#  Arithmetic Expansion

echo $(($(cat allAVA.jpgl | wc -l) /3))


# calculator

bc -l

# how to use man

man 3

1 general commands 
2 system calls ( specially for C)
3 library functions
4 special files 
5 file formats and conventions
6 Games
7 Mis 
8 Sys admin and daemons

# remove the first 6 lines

tail -n +6 file 

# copy files and their directory tree

rsync -a --include '*/' --include '*.mp3' --exclude '*' source/ target/

This will (1) include any directory or subdirectory, whatever its contents, (2) include all files with an mp3 extension, and (3) exclude all other files. Any number of other file types can be included by adding their one include pattern for each before the --exclude, e.g. --include '*.bmp'.

# date tricks

date +%W  # gives the week number

date --date='2 days ago'


Print the day of year of Christmas in the current year:

$ date --date='25 Dec' +%j




# split and rename

        split -d -l 30000 allAVA.jpgl allAVA                                                                  
        for s in `seq -f "%03g" $(NSPLIT) `; do \                                                             
                mv allAVA$${spl}{,.jpgl};done \                                                               
        done;        



NDIGITS=4                                                                                                     
$(PATH_LISTS)/splits.log:                                                                                     
        split -a $(NDIGITS) -l $$(($$(cat $(PATH_LISTS)/$(CORPUS).jpgl | wc -l) /$(NSPLIT)+1))  -d $(PATH_LIS\
TS)/$(CORPUS).jpgl $(PATH_LISTS)/$(CORPUS)                                                                    
        for s in  $(PATH_LISTS)/$(CORPUS)0*; do \                                                             
                mv $${s}{,.jpgl}; \                                                                           
        done; > $@ 




# braces expansion, create multiple directories at once


mkdir {dir1,dir2,dir3}


mkdir prefix_{dir1,dir2,dir3}


http://wiki.bash-hackers.org/syntax/expansion/brace


# shows file.txt on stdout and save it at the same time in log.txt

cat file | tee log.txt

# parallel ssh

/opt/STools/python/2.6-x86_64/bin/pssh -h ~/bin/spiders.pssh -o ./cluster.log  " df -h | grep local"

where
spiders.pssh

is a file containing all the servers addresses 

# check if a list of files exists

if [ -e "/directory/file.txt" ]; then \  
echo "[FILE]: exists"; \ 
else \
echo "[FILE]: NOT exists"; \ 
fi

if [ ! -f /tmp/foo.txt ];then    echo "File not found!"fi

# get the second value from std input and convert it into a number

counter =($2)

# alternative way to loop over a file lines

while read line; do \ 
echo -e "$line"; \
done; < filename.txt 


# erase newline, backspace

tr -d '\n'

# all about screen

0. ssh yourserver

1. launch a screened command:

screen -S   

get out of here

2. list active screens

screan -ls 

3. reattach the session

screen -r

4. get rid of the dead session

screen -wipe

screen -x

# using tabs or colors with echo requires the option -e

GREEN_S=\e[00;32m                                                                                                    

GREEN_E=\e[00m:                                                                                                      
                                                                                                                                                         
echo -e "$(GREEN_S)PATH_LISTS\t: $(PATH_LISTS)$(GREEN_E)"                                                    
echo -e "$(GREEN_S)PATH_RESULTS\t: $(PATH_RESULTS)$(GREEN_E)"   


#  [TO VERIFY] Difference between

1. PQFILE=$(echo $(PATH_FEATURES) | sed -e 's@hst@@')/kmeans.pq

and

2. PQFILE=`echo $(PATH_FEATURES) | sed -e 's@hst@@'`/kmeans.pq


1. is executed when variable is defined , 2. is executed everytime the variable is used

# pick randomly ( be careful, this option is available only on most recent versions of sort)

sort -R 

if statatement in bash:

if [ "$(hostname)" != '' ]; then                                                              

                                                                                                         

    alias ipython='/opt/STools/python/2.6-x86_64/bin/ipython -pylab -gthread'    

                                                                                                                                  

else

                                                                                                                               

    alias ipython='/opt/STools/python/2.4.3/bin/ipython -pylab -gthread'                                                                                                                                     

fi   

# if statement with variable increment 

if [ $1  -eq 0 ] 
then 
  echo "Input is zero"
  echo" and the sum between 1 and 77 is $((1+77))"
fi

# smbliclient command line 


$ smbclient ///remote/dir -U luca -p –n

# All binaries sources and compilation files are in /usr/share !!! 

( remember the subversion mess!)