Shell Scripts & Linux Programming

What is Shell?

# CLI environment: SH (basic shell), BASH (common Linux shell language), Ksh, Csh, Tsh, Zsh (recommended).
# It is a program managing the commands in the command line.
# configuration file : .bashrc

# install a new shell
[mo@mos-computer ~]$ sudo yum install ksh
# change shell
[mo@mos-computer ~]$ chsh
Changing shell for mo.
New shell [/bin/bash]: /bin/ksh
Password: 
Shell changed.

# coding in bash shell scripts
[mo@mos-computer ~]$ vim test.sh
[mo@mos-computer ~]$ cat test.sh
#!/bin/bash

# List all files in folder
ls

[mo@mos-computer ~]$ chmod +x test.sh
[mo@mos-computer ~]$ ./test.sh
compression	       myFIle		   rar
Desktop		       name_sorted.txt	   rarlinux-x64-5.7.0.tar.gz
Documents	       name.txt		   README
Downloads	       newfile		   redirect
errors.log	       new_folder	   repeat.txt
file_changed_name.txt  newly_created_file  results.txt
file-copy.txt	       node		   Templates
grep_log	       nohup.out	   test
htop-2.2.0	       number.txt	   test.sh
htop-2.2.0.tar.gz      output_find	   unique.txt
Music		       Pictures		   Videos
myFile		       Public

# bash debugger
[mo@mos-computer ~]$ bash -x test.sh
+ pwd
/home/mo
+ ls
compression	       myFIle		   rar
Desktop		       name_sorted.txt	   rarlinux-x64-5.7.0.tar.gz
Documents	       name.txt		   README
Downloads	       newfile		   redirect
errors.log	       new_folder	   repeat.txt
file_changed_name.txt  newly_created_file  results.txt
file-copy.txt	       node		   Templates
grep_log	       nohup.out	   test
htop-2.2.0	       number.txt	   test.sh
htop-2.2.0.tar.gz      output_find	   unique.txt
Music		       Pictures		   Videos
myFile		       Public

# also you can customize a command and add it to PATH
[mo@mos-computer ~]$ echo $PATH
/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:/home/mo/.local/bin:/home/mo/bin
[mo@mos-computer ~]$ sudo cp test.sh /usr/bin
[sudo] password for mo: 
[mo@mos-computer ~]$ test.sh
/home/mo
compression	       myFIle		   rar
Desktop		       name_sorted.txt	   rarlinux-x64-5.7.0.tar.gz
Documents	       name.txt		   README
Downloads	       newfile		   redirect
errors.log	       new_folder	   repeat.txt
file_changed_name.txt  newly_created_file  results.txt
file-copy.txt	       node		   Templates
grep_log	       nohup.out	   test
htop-2.2.0	       number.txt	   test.sh
htop-2.2.0.tar.gz      output_find	   unique.txt
Music		       Pictures		   Videos
myFile		       Public

Variables in Shell

[mo@mos-computer ~]$ vim variable.sh
[mo@mos-computer ~]$ chmod +x variable.sh 
[mo@mos-computer ~]$ ./variable.sh
#echo: show info in the shell , -e change line
[mo@mos-computer ~]$ echo 'Fuck'
Fuck
[mo@mos-computer ~]$ echo -e 'first\nsecond'
first
second
[mo@mos-computer ~]$ vim variable.sh 
[mo@mos-computer ~]$ ./variable.sh 
Hello, it's me
[mo@mos-computer ~]$ cat variable.sh 
#!/bin/bash

message=$'Hello, it\'s me'
echo $message

# see what happened for the following special case ('', "" and ``)
[mo@mos-computer ~]$ ./variable.sh 
Hi Jen, Hello, it's me
[mo@mos-computer ~]$ cat variable.sh 
#!/bin/bash

message=$'Hello, it\'s me'
echo "Hi Jen, $message"
[mo@mos-computer ~]$ vim variable.sh 
[mo@mos-computer ~]$ ./variable.sh 
Hi Jen, Hello, it's me
You are in the /home/mo
[mo@mos-computer ~]$ cat variable.sh 
#!/bin/bash

message=$'Hello, it\'s me'
echo "Hi Jen, $message"

message1=`pwd`
echo "You are in the $message1"

# read
[mo@mos-computer ~]$ vim variable.sh 
[mo@mos-computer ~]$ ./variable.sh 
mo
Hi Jen, Hello, it's me, mo
You are in the /home/mo
[mo@mos-computer ~]$ cat variable.sh 
#!/bin/bash
read name

message=$'Hello, it\'s me,'
echo "Hi Jen, $message $name"

message1=`pwd`
echo "You are in the $message1"
# assign value to multiple vars, split them with spaces
[mo@mos-computer ~]$ ./variable.sh 
shuheng mo
Hi Jen, Hello, it's me, shuheng mo
You are in the /home/mo
[mo@mos-computer ~]$ cat variable.sh 
#!/bin/bash
read firstname lastname

message=$'Hello, it\'s me,'
echo "Hi Jen, $message $firstname $lastname"

message1=`pwd`
echo "You are in the $message1"

# other useful args
[mo@mos-computer ~]$ vim variable.sh 
[mo@mos-computer ~]$ cat variable.sh 
#!/bin/bash
read -p 'Please enter your full name (30 characters max):' -n 30 firstname lastname

message=$'Hello, it\'s me,'
echo -e "Hi Jen, $message \n $firstname $lastname"

message1=`pwd`
echo "You are in the $message1"

read -p 'Enter the code to defuse the bomb, you have 5 sec!' -t  5 code
echo -e '\nBoom!!!!'

read -p 'Great hero, you saved the world! Now tell me your age:' -s age
echo -e "\nYeah, I will tell every one you are $age, you old fool haha!"
[mo@mos-computer ~]$ ./variable.sh 
Please enter your full name (30 characters max):shuheng mo
Hi Jen, Hello, it's me, 
 shuheng mo
You are in the /home/mo
Enter the code to defuse the bomb, you have 5 sec!12

Boom!!!!
Great hero, you saved the world! Now tell me your age:
Yeah, I will tell every one you are 18, you old fool haha!

Arithmetic operations in bash

# in bash, all vars are 'String'
[mo@mos-computer ~]$ chmod +x arithmetic.sh 
[mo@mos-computer ~]$ ./arithmetic.sh 
c = 11
[mo@mos-computer ~]$ cat arithmetic.sh 
#!/bin/bash

let "a = 5"
let "b = 6"
let "c = a + b"

echo "c = $c"
# environment
[mo@mos-computer ~]$ env
XDG_VTNR=1
XDG_SESSION_ID=1
SSH_AGENT_PID=2404
HOSTNAME=mos-computer
IMSETTINGS_INTEGRATE_DESKTOP=yes
HOST=mos-computer
XDG_MENU_PREFIX=gnome-
SHELL=/bin/bash
TERM=xterm-256color
VTE_VERSION=5204
HISTSIZE=1000
GNOME_TERMINAL_SCREEN=/org/gnome/Terminal/screen/9fa9ced3_316a_46fb_8eaa_9f639d94a847
IMSETTINGS_MODULE=none
GROUP=mo
USER=mo
DESKTOP_AUTOSTART_ID=10f93cf49a8ee6482f161545299817421600000022720007
LS_COLORS=rs=0:di=38;5;27:ln=38;5;51:mh=44;38;5;15:pi=40;38;5;11:so=38;5;13:do=38;5;5:bd=48;5;232;38;5;11:cd=48;5;232;38;5;3:or=48;5;232;38;5;9:mi=05;48;5;232;38;5;15:su=48;5;196;38;5;15:sg=48;5;11;38;5;16:ca=48;5;196;38;5;226:tw=48;5;10;38;5;16:ow=48;5;10;38;5;21:st=48;5;21;38;5;15:ex=38;5;34:*.tar=38;5;9:*.tgz=38;5;9:*.arc=38;5;9:*.arj=38;5;9:*.taz=38;5;9:*.lha=38;5;9:*.lz4=38;5;9:*.lzh=38;5;9:*.lzma=38;5;9:*.tlz=38;5;9:*.txz=38;5;9:*.tzo=38;5;9:*.t7z=38;5;9:*.zip=38;5;9:*.z=38;5;9:*.Z=38;5;9:*.dz=38;5;9:*.gz=38;5;9:*.lrz=38;5;9:*.lz=38;5;9:*.lzo=38;5;9:*.xz=38;5;9:*.bz2=38;5;9:*.bz=38;5;9:*.tbz=38;5;9:*.tbz2=38;5;9:*.tz=38;5;9:*.deb=38;5;9:*.rpm=38;5;9:*.jar=38;5;9:*.war=38;5;9:*.ear=38;5;9:*.sar=38;5;9:*.rar=38;5;9:*.alz=38;5;9:*.ace=38;5;9:*.zoo=38;5;9:*.cpio=38;5;9:*.7z=38;5;9:*.rz=38;5;9:*.cab=38;5;9:*.jpg=38;5;13:*.jpeg=38;5;13:*.gif=38;5;13:*.bmp=38;5;13:*.pbm=38;5;13:*.pgm=38;5;13:*.ppm=38;5;13:*.tga=38;5;13:*.xbm=38;5;13:*.xpm=38;5;13:*.tif=38;5;13:*.tiff=38;5;13:*.png=38;5;13:*.svg=38;5;13:*.svgz=38;5;13:*.mng=38;5;13:*.pcx=38;5;13:*.mov=38;5;13:*.mpg=38;5;13:*.mpeg=38;5;13:*.m2v=38;5;13:*.mkv=38;5;13:*.webm=38;5;13:*.ogm=38;5;13:*.mp4=38;5;13:*.m4v=38;5;13:*.mp4v=38;5;13:*.vob=38;5;13:*.qt=38;5;13:*.nuv=38;5;13:*.wmv=38;5;13:*.asf=38;5;13:*.rm=38;5;13:*.rmvb=38;5;13:*.flc=38;5;13:*.avi=38;5;13:*.fli=38;5;13:*.flv=38;5;13:*.gl=38;5;13:*.dl=38;5;13:*.xcf=38;5;13:*.xwd=38;5;13:*.yuv=38;5;13:*.cgm=38;5;13:*.emf=38;5;13:*.axv=38;5;13:*.anx=38;5;13:*.ogv=38;5;13:*.ogx=38;5;13:*.aac=38;5;45:*.au=38;5;45:*.flac=38;5;45:*.mid=38;5;45:*.midi=38;5;45:*.mka=38;5;45:*.mp3=38;5;45:*.mpc=38;5;45:*.ogg=38;5;45:*.ra=38;5;45:*.wav=38;5;45:*.axa=38;5;45:*.oga=38;5;45:*.spx=38;5;45:*.xspf=38;5;45:
GNOME_TERMINAL_SERVICE=:1.103
HOSTTYPE=x86_64-linux
SSH_AUTH_SOCK=/run/user/1000/keyring/ssh
SESSION_MANAGER=local/unix:@/tmp/.ICE-unix/2272,unix/unix:/tmp/.ICE-unix/2272
USERNAME=mo
GNOME_SHELL_SESSION_MODE=classic
DESKTOP_SESSION=gnome-classic
MAIL=/var/spool/mail/mo
PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:/home/mo/.local/bin:/home/mo/bin
QT_IM_MODULE=ibus
PWD=/home/mo
XDG_SESSION_TYPE=x11
XMODIFIERS=@im=ibus
EDITOR=nano
LANG=en_US.UTF-8
GDM_LANG=en_US.UTF-8
GDMSESSION=gnome-classic
HISTCONTROL=ignoredups
SHLVL=4
XDG_SEAT=seat0
HOME=/home/mo
OSTYPE=linux
GNOME_DESKTOP_SESSION_ID=this-is-deprecated
VENDOR=unknown
MACHTYPE=x86_64
LOGNAME=mo
XDG_SESSION_DESKTOP=gnome-classic
DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-EtJoG9tnzO,guid=449c25136c054a6ce9840dfc6049db45
XDG_DATA_DIRS=/home/mo/.local/share/flatpak/exports/share/:/var/lib/flatpak/exports/share/:/usr/local/share/:/usr/share/
LESSOPEN=||/usr/bin/lesspipe.sh %s
WINDOWPATH=1
DISPLAY=:0
XDG_RUNTIME_DIR=/run/user/1000
XDG_CURRENT_DESKTOP=GNOME-Classic:GNOME
XAUTHORITY=/run/gdm/auth-for-mo-pe9rGs/database
COLORTERM=truecolor
_=/usr/bin/env

#shell var
[mo@mos-computer ~]$ vim shell.sh
[mo@mos-computer ~]$ chmod +x shell.sh 
[mo@mos-computer ~]$ ./shell.sh
Your default system shell is /bin/bash
[mo@mos-computer ~]$ cat shell.sh
#!/bin/bash

echo "Your default system shell is $SHELL"

# export command
# args vars, $# --> args num; $0 --> running scripts num; $N --> Nth args
[mo@mos-computer ~]$ vim shell.sh 
[mo@mos-computer ~]$ ./shell.sh param1 param2 param3
Your default system shell is /bin/bash
You have executed ./shell.sh, there are 3 parameters
The first parameters is param1
[mo@mos-computer ~]$ cat shell.sh 
#!/bin/bash

echo "Your default system shell is $SHELL"
echo "You have executed $0, there are $# parameters"
echo "The first parameters is $1"

# shift was used in the loop to make sure the params were executed one by one
[mo@mos-computer ~]$ ./shell.sh param1 param2 param3
Your default system shell is /bin/bash
You have executed ./shell.sh, there are 3 parameters
The first parameters is param1
The first parameters is now go to param2
[mo@mos-computer ~]$ cat shell.sh 
#!/bin/bash

echo "Your default system shell is $SHELL"
echo "You have executed $0, there are $# parameters"
echo "The first parameters is $1"
shift
echo "The first parameters is now go to $1"
#array
[mo@mos-computer ~]$ vim array.sh 
[mo@mos-computer ~]$ ./array.sh 
value2
[mo@mos-computer ~]$ cat array.sh 
#!/bin/bash

array=('value0' 'value1' 'value2')
array[5]='value5'
echo ${array[2]}

# note that idx start from 0

conditional structure?

# Notice the space in the condition position
[mo@mos-computer ~]$ cat condition.sh 
#!/bin/bash

name1="Mo"
name2="Jen"

if [ $name1 = $name2 ]
then
echo "Wow, you two have the same name!"
fi

# then we will need a else and then elif
if [CONDITION]
then
	DO STH...
elif [CONDITION]
then
	DO STH...
else
	DO STH...
fi

#condition debugger : String , decimals, files
# $string1 = $string2, note that Shell is case sensitive
# $string1 != $string2
# -z $string, string is NULL?
# -n $string, string is not NULL?
[mo@mos-computer ~]$ vim condition1.sh 
[mo@mos-computer ~]$ ./condition1.sh 
No parameter
[mo@mos-computer ~]$ ./condition1.sh 1
There is more than one parameter
[mo@mos-computer ~]$ cat condition1.sh 
#!/bin/bash

if [ -z $1 ]
then
	echo "No parameter"
else
	echo "There is more than one parameter"
fi

# you can test number too, do some search on this
#!/bin/bash
if [ $1 -ge 10 ]
then
    echo "You have entered a number greater than 10 or equal to 10"
else
    echo "You have entered a number lower than 10"
fi

# you can see more things like this in /mnt/hgfs/tutorial/code/shell/condition
# there are too many condition cases you can write, do some practice

#case
[mo@mos-computer ~]$ vim case.sh
[mo@mos-computer ~]$ chmod +x case.sh 
[mo@mos-computer ~]$ ./case.sh 
Sorry, I do not know who the fuck you are.
[mo@mos-computer ~]$ ./case.sh Mark
Hello Mark!
[mo@mos-computer ~]$ vim case.sh 
[mo@mos-computer ~]$ cat case.sh 
case $1 in
	"Mark")
		echo "Hello Mark!"
		;;
	"Luke")
		echo "Hello Luke!"
		;;
	"John")
		echo "Hello John"
		;;
	*)
		echo "Sorry, I do not know who the fuck you are."
		;;
esac

# see more options in code folder

#Loop

#while
[mo@mos-computer ~]$ vim loop.sh
[mo@mos-computer ~]$ chmod +x loop.sh 
[mo@mos-computer ~]$ ./loop.sh 
Say yes : yes
[mo@mos-computer ~]$ ./loop.sh 
Say yes : no
Say yes : no
Say yes : no
Say yes : ^C
[mo@mos-computer ~]$ cat loop.sh 
#!/bin/bash

while [ -z $response ] || [ $response != 'yes' ]
do
	read -p 'Say yes : ' response
# until
# for
[mo@mos-computer ~]$ vim loop1.sh
[mo@mos-computer ~]$ chmod +x loop1.sh 
[mo@mos-computer ~]$ ./loop1.sh 
File found : arithmetic.sh
File found : array.sh
File found : case.sh
File found : compression
File found : condition1.sh
File found : condition.sh
File found : Desktop
File found : Documents
File found : Downloads
File found : errors.log
File found : file_changed_name.txt
File found : file-copy.txt
File found : grep_log
File found : htop-2.2.0
File found : htop-2.2.0.tar.gz
File found : loop1.sh
File found : loop.sh
File found : Music
File found : myFile
File found : myFIle
File found : name_sorted.txt
File found : name.txt
File found : newfile
File found : new_folder
File found : newly_created_file
File found : node
File found : nohup.out
File found : number.txt
File found : output_find
File found : Pictures
File found : Public
File found : rar
File found : rarlinux-x64-5.7.0.tar.gz
File found : README
File found : redirect
File found : repeat.txt
File found : results.txt
File found : shell.sh
File found : Templates
File found : test
File found : test.sh
File found : unique.txt
File found : variable.sh
File found : Videos
[mo@mos-computer ~]$ cat loop1.sh 
#!/bin/bash

listfile=`ls`

for file in $listfile
do
	echo "File found : $file"
done

#classic for loop
[mo@mos-computer ~]$ for i in `seq 1 10`
> do
> echo $i
> done
1
2
3
4
5
6
7
8
9
10

Shell functions

[mo@mos-computer ~]$ chmod +x function.sh 
[mo@mos-computer ~]$ ./function.sh 
Hello, I am a function.
Hello, I am a function.
[mo@mos-computer ~]$ cat function.sh 
#!/bin/bash

print_something (){
	echo "Hello, I am a function."
}

print_something
print_something

# passing args to func
[mo@mos-computer ~]$ vim function.sh 
[mo@mos-computer ~]$ ./function.sh 
Hello, I am Mo.
Hello, I am Jen.
[mo@mos-computer ~]$ cat function.sh 
# !/bin/bash

print_something (){
	echo "Hello, I am $1."
}

print_something Mo
print_something Jen

# return a ... only status available
# see more options in the code

# local and global variables

#!/bin/bash
local_global () {
    local var1='local 1'
    echo Inside function: var1 is $var1 : var2 is $var2
    var1='changed again'   # 这里的 var1 是函数中定义的局部变量
    var2='2 changed again' # 这里的 var2 是函数外定义的全局变量
}

var1='global 1'
var2='global 2'

echo Before function call: var1 is $var1 : var2 is $var2

local_global

echo After function call: var1 is $var1 : var2 is $var2

#reload a command
#!/bin/bash

ls () {
    command ls -lh
}

ls

Shell (image) to display webpage

[mo@mos-computer gallery]$ cat gallery.sh
#!/bin/bash

# Verification of parameter（验证参数）
# If no parameter, use a default value（如果没有给出参数，那么用默认值 gallery.html ）
if [ -z $1 ]
then
    output='gallery.html'
else
    output=$1
fi

# Preparation of files and folders（准备文件和目录）
echo '' > $output

if [ ! -e thumbnails ]
then
    mkdir thumbnails
fi

# Beginning of HTML（HTML 文件的开头）
echo '<!DOCTYPE html>
<html>
  <head>
    <title>My Gallery</title>
  </head>
  <body>
    <p>' >> $output

# Generation of thumbnails and the HTML web page（生成图片的缩略图和 HTML 的页面主体）
for image in `ls *.jpg *.png *.jpeg *.gif 2>/dev/null`
do
    convert $image -thumbnail '200x200>' thumbnails/$image
    echo '      <a href="'$image'"><img src="thumbnails/'$image'" alt=""/></a>' >> $output
done

# End of HTML（HTML 文件的结尾）
echo '    </p>
  </body>
</html>' >> $output

git clone [URL]

Shell for statistics

[mo@mos-computer words]$ grep -io a words.txt
[mo@mos-computer words]$ grep -i a words.txt | wc -l
206518
[mo@mos-computer words]$ grep -io a words.txt | wc -l
273400
[mo@mos-computer words]$ for char in {a..z};do
> grep -io "$char" words.txt | wc -l
> done
273400
61975
146090
108173
363325
38763
80206
86490
297800
5073
25618
187617
99955
242172
241766
109040
5696
237931
245420
223998
126597
32467
22053
10112
67946
13932
[mo@mos-computer words]$ cat statistics.sh 
#!/bin/bash

# Verification of parameter
# 确认参数
if [ -z $1 ]
then
    echo "Please enter the file of dictionary !"
    exit
fi

# Verification of file existence
# 确认文件存在
if [ ! -e $1 ]
then
    echo "Please make sure that the file of dictionary exists !"
    exit
fi

# Definition of function
# 函数定义
statistics () {
  for char in {a..z}
  do
    echo "$char - `grep -io "$char" $1 | wc -l`" | tr /a-z/ /A-Z/ >> tmp.txt
  done
  sort -rn -k 2 -t - tmp.txt
  rm tmp.txt
}

# Use of function
# 函数使用
statistics $1
[mo@mos-computer words]$ ./statistics.sh words.txt
E - 363325
I - 297800
A - 273400
S - 245420
N - 242172
O - 241766
R - 237931
T - 223998
L - 187617
C - 146090
U - 126597
P - 109040
D - 108173
M - 99955
H - 86490
G - 80206
Y - 67946
B - 61975
F - 38763
V - 32467
K - 25618
W - 22053
Z - 13932
X - 10112
Q - 5696
J - 5073