Unix Goodies
perl
# for any line in the file test.txt containing ‘host-name foo’ add a line ‘URL /wiki/foo’ with the same indentation
perl -pi -e 's/^(\s*)(host-name\s+)(\S+)$/\1\2\3\n\1URL \/wiki\/\3/; test.txt'
# fast check if perl has a certain module locally available
perl -MNet::Telnet -e 1
# delete all LF/CR in a text
perl -i -pe 'chomp unless eof' test.txt
# query version of an installed module in perl
perl -MText::Autoformat -e 'print $Text::Autoformat::VERSION'
# how to replace only the first instance of a match using perl
perl -pi -e '!$found && s|^MANDATORY_MANPATH|MANDATORY_MANPATH\t/client/man\nMANDATORY_MANPATH| && ($found = 1)' /etc/manpath.config
# strip HTML in a very crude way using perl
perl -p -i -e 's/]*?)>//g' file
# convert something from uppercase to lowercase using perl
perl -pi -e 's/^Domain: (.*)\.eu/"Domain: ".lc($1).".eu"/ge' file
# strip trailing whitespaces using perl
perl -p -i -e 's/\s+$/\n/;' file
# debug perl programs non-interactivly
PERLDB_OPTS="Auto Non Line=prog.out" perl -d debugme.pl
# extract everything between to tags in perl
perl -p -0777 -e 's|.*(.*).*|\1|s' comments.txt
# replace a character with a new line in perl
perl -p -i -e 's/\//\n/g;' DATEI
# change all IP Adresses in one file to one specific
perl -p -e -i 's/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/4.5.6.7/g;' xyno
# how to change tags in html from upper to lowercase using perl
perl -pi -e 's/]+)>/""/ge' foo.html
# how to say RTFM in a nice way
perl -e 'print pack(c5,(41*2),sqrt(7056),(unpack(c,H)-2),oct(115),10);'
# replace tags within the same file(s) using perl
perl -p -i -e 's/something/else/;' `cat yourlistoffiles`
# replace soa with new value using perl
perl -p -i -e 's/\d{9}/200311050/g;' $i
# remove newlines with perl
echo -e "foo\nbar\nbaz\n"|perl -pe "s/\n/ /"
# keyword completition in perl
control+n
mencoder
# cut a certain part of an avi (start at 10 min, length 2 min)
mencoder -ss 10:00 -endpos 2:00 numbers.avi -oac copy -ovc copy -o numbers.sample.avi
# convert to avi with xvid and mp3
mencoder -oac mp3lame -ovc lavc -lavcopts vcodec=xvid:acodec=mp3 video.mov -o video.avi
mencoder -oac mp3lame -ovc lavc -lavcopts vcodec=xvid:vbitrate=1200:abitrate=128 -vf scale -zoom -xy 652
# scale video – set width, height is calculated by aspect ratio of source)
mencoder -vf scale -zoom -xy 640 -oac mp3lame -ovc lavc -lavcopts vcodec=xvid:acodec=mp3 video.mov -o video.avi
mplayer
# detect crop area (no output, start at 3 minutes)
mplayer video.vob -vo null -ao null -ss 180 -vf cropdetect
# take screenshots (press ’s’, outputs shot####.png to working dir)
mplayer -vf screenshot video.avi
# show media information with mplayer
mplayer -identify -vc null -ac null -frames 0
# dump mpeg 4 audio as pcm wav
mplayer -ao pcm -vc dummy -vo null FILE.m4a
vi/vim
# Regular Expression to remove preceding whitespaces in vi:
:1,$ s/^ *//
# remove trailing whitespaces in vim
:%s/\s\+$//
# collaps consecutive white lines into one in vim
:v/./.,/./-1join
# delete blank lines in vim
:g/^\s*$/d
# show all special characters in vim
:set list
# add spellchecking to vim 7.0
:set spell
# replace one plattforms carridge return by the current one with vim
:%s/\r/\r/g
# expand tabs to whitespace in vim
:set expandtab
:retab
# remove control-m in vi
:%s/^V^M//g
# show all special charactrers in vim
:set list
# indent each line by one tab in vim
:1,$>
# make vim use colours like mutt
:colorscheme darkblue
# avoid indenting when pasting into vim
set paste
LaTeX
# directly create a pdf from a tex source
\usepackage{ae}
pdflatex foobar.tex
# comments in latex
\usepackage{comment}
\begin{comment} \end{comment}
(g)awk/sed
# Format Textfile to a fixed 80 char width limit using awk
awk '{ printf "%80s\n", $0}' infile > outfile
# how to add fields with awk
awk -F ":" '{ x += $7 }; END { print x}' nomicon
# how to wrap urls within html using sed
sed 's/.*/<a href="&">&/'
# erase leading whitespaces with sed
sed "s#^ *\([^0-9]*\)#\1#g"
# search a strink and output a field using awk
awk -F":" '/swadm/ { print $3 }' /etc/group
# print first line and all lines matching search term
gawk 'NR==1 || /John/' file
# join a record on n lines seperated by an emtpy line to one
gawk '$1=$1' RS= foobar
# print a paragraph containing LIHAS using sed
sed -e '/./{H;$!d;}' -e 'x;/LIHAS/!d' flatfile
# split up data into multipe files, key is the first column with awk
awk '{print > "Out"$1".dat"}'
# delete trailing whitespaces with sed
sed 's/[ \t]*$//' -i file
# print text between 2 regexes with sed
sed -n "/admin-c/,/Source:/p"
# get last and next to last element with awk
awk '{print $(NF-1), $NF}' file
# delete empty lines with sed
sed '/./!d' -i $i
# use sed as grep
sed -e '/^RT/q;d' file
sed -n "/^RT/p" file
# get rid of [] on lines with sed
sed s/[][]//g
# convert unixtime to date using awk
date +%s | awk '{ print strftime("%T", $1 ) }'
netstat
# get an extended list of open tcp and udp ports and processes with pid listening on them
netstat -tulpen
# see the network packet throughput
netstat -I fx0 -w 3
# netstat big output
netstat -s -c -n
# check to which user an open port belongs using netstat
netstat -at
tcpdump
# how to tcpdump,
tcpdump -i znb1 host 212.9.189.121 and not port ssh and not host monitor.lf.net
# how to tcpdump and save the output to a file
tcpdump -l -i fxp1 212.9.189.121 | tee -a /tmp/foofo
# debug nameserver queries via tcpdump
tcpdump -s512 -i fxp0 udp port domain
ex/exim
# how to join lines with ex
ex products.xml +"g/z:row/j" +"g/z:row/j" +"wq"
# replace soa with new value using ex
ex +"s/[0-9]\{9\}/200101019/g" +"wq" $i
# short statistics from eximstats
eximstats -t0 -q0 -h0 -ne -nr /var/log/exim/main*
# send mail using exim
exim -t < file
# run exim in queuing only mode
/client/bin/exim -bd -odq
grep
# list amount of mails in the mbox using egrep
egrep -c "^From .*[0-9]+:[0-9]+:[0-9]+" Mail/inbox
# get one specific field using cut
grep swadm /etc/group | cut -d : -f 3
# clean comment cluttered configuration files using egrep
egrep -v "^#|^$" file
# find seconary zones from a specific master
grep -v ^# conf.sec | gawk '$1=$1' RS= | grep 194.233.145.1 | cut -d'"' -f2
# grep network traffic for bare words
ngrep -iq 'Yagger' tcp port 6667
sort
# sort a file using multiple keys,sort field 11 numerically with sort
sort -k1,1 -k11,11n file
# sort IPs numerically using sort
sort -n -t. -k1,1 -k2,2 -k3,3 -k4,4 ip.liste
# sort a domain list by tld using sort
sort -t. -k2
mount
# how to mount iso images
Linux: mount -t iso9660 -r -o loop texlive-20010126.iso /mnt
SunOS: mount -r -F fbk -o type=hsfs /dev/fbk0:texlive-20010126.iso /mnt
# mount kodak photo cd
mount_cd9660 -o ro -v -s 0 /dev/cdrom /cdrom
date
# how do display unix time with date
date +%s
# rfc 2822 compliant date
date -R
# show localtime in UTC
date -u
openssl
# encrypt with the symmetric blowfish cipher using openssl
openssl bf -in foobar -out foobar.enc
# decrypt with the symmetric blowfish cipher using openssl
openssl bf -d -in foobar.enc -out foobar.dec
# check certificate of an https website using openssl
openssl s_client -connect :443 -CAfile /usr/local/lib/openssl/certs/ca-bundle.crt | openssl x509 -text | less
# verify a s/mime signature
openssl smime -verify -in messagefile >/dev/null
# extract the s/mime Certificate to something usable 
openssl smime -pk7out -in messagefile | openssl pkcs7 -print_certs
find
# recursively delete all svn subdirectories
find . -depth -type d -name ".svn" -execdir rm -rf {} \;
# recursively find Python files containing prettyprint, ignore case [i], print matching lines prefixed with filename [H] and linenumber [n]
find . -name "*.py" -exec grep -nHi prettyprint '{}' \;
# search /sw for known dead symlinks and delete them
find . -printf "%p %l\n" | awk '/tetex-3/ {print $1}' | xargs rm -v
# how to use find and exec
find . -name "*.html" -exec dos2unix --d2u {} \;
# find files and copy them elsewhere
find . -name "*.tar.gz" | xargs -i /bin/sh -c "cp -v {} /tmp/backup/"
# find dead symlinks and delete them with find and rm
for i in $(find . -type l); do test -e $i || rm -v /client/$i; done
mp3/ogg/wav
# convert ogg files to mp3s using ogg123
ogg123 -d wav -f - "$i" | lame - "${i%.ogg}.mp3"
# How to convert mp3s to wav using lame
for i in *.mp3; do lame --decode "$i"; done
# convert a wav file to the raw format (audio cd, burncd)
sox -t wav -r 44100 -s -w -c 2 track.wav track.raw
file suffix
# remove file suffix with bash
Rest=${Datei%.*}
# rename suffixes using bash
for i in *.gif ; do convert "$i" "${i%.gif}.png"
# strip suffix within bash
${i%.jpg}.ps
# get the file suffix using rev and cut
echo BioHazard.gif | rev | cut -d . -f 1 | rev
whois
# strip notice from the whois output for CNO domains
/usr/bin/whois lf.net | sed '/NOTICE/,/policy/d'
# get all IP Prefixes for one ASN
whois -h whois.radb.net \!gasASN
# get all objects which have a MNT via whois3
whois3 -B -h whois.ripe.net -i mb LFNET-MNT
# find the whois server for a tld via dig
dig +short edu.whois-servers.net CNAME
# How to grab stderr if redirectiong shell script output to a file
program >>file 2>&1
# Read a file line by line and process line:
while read line
do
echo "$line"
done <inputfile.txt
# Determine the user:
# (depends on system, sunos only likes id without parameters):
whoami
id -nu
id
# Get current shell:
ps -p $$
# alternative
echo $0
# multirename using mv
for f in *.txt; do mv ./"$f" "${f%txt}htm"; done
# how to print doublesided and nice in book format
psbook dokument.ps | psnup -2 | psselect -o | lpr
psbook dokument.ps | psnup -2 | psselect -e -r | lpr
# how to strip control ^M in a neat way using tr
tr -d '15' newfile
# How to copy directories in a neat way using tar
tar zcpBf ./dir1 - |(cd somewhere; tar zxpBvf -)
# How to set TTY manually if not set by system:
TTY=$(tty) && TTY=${TTY##/dev/}
# Shared Memory wieder freigeben wenn gdk einen shmget fail meldet:
# mit ipcs ermitteln was so an shared memory weg ist
ipcrm -m ID
# how to do a ssh tunnel
ssh -C -L 1812:localhost:1812 root@somehost
# how to have the path in the xtitle
export PROMPT_COMMAND="xtitle \$(pwd)"
# counting in bash
for (( i=300;i>-1;i-- )); do sleep 1;echo 300; done
# show biggest directories within the mountpoint using du
du -x | sort -nr | less
# take down an if alias
ifconfig de0 -alias 212.9.174.20 netmask 255.255.255.255
# russian bash roulette, replace echo PENG with „rm -rf /“
if [ $(($RANDOM%6)) -eq 0 ]; then echo PENG; fi
# how to view a certain revision from RCS
co -p1.3
# how to restore a file from RCS
co -r1.3
# how to make a screenshot using import
import -window root screenshot.png
# startx without listening to Port 6000 and have dpi set to 100
X -nolisten -dpi 1000tcp
# single user mode in freebsd, no uptime loss, use exit to go multi
shutdown now
# find out what is listening on using lsof
lsof -i :1851
# see to where some IP is routed
route get 212.9.190.1
# make xterm usable after scp
stty sane
# extract an audio track using dd
dd if=/dev/acd0t1 of=track1.cdr conv=swab bs=2352
# calculate hours from seconds using bc
echo "obase=60; 264" | bc | xargs | tr ' ' :
# see which programs use which socket on FreeBSD
sockstat -4
# read manual pages directly using groff
groff -mandoc -Tlatin1 wget.1
# show information about a serial line, cisco
show interface ser 2/0
# show information about the X server settings
xset q
# use bash to convert non-decimal values into decimals
echo $((16#100))
# convert between numbers of different bases using bc
echo 'obase=16;ibase=2;10' | bc
# solaris system information
/usr/sbin/psrinfo -v; /usr/sbin/prtconf; /usr/platform/`uname -i`/sbin/prtdiag -v
# get xcb contents on the shell
xcb -p 3
# write to the xcb buffer
echo "Foobar " | xcb -s 1
# change the keyboard layout in x
setxkbmap -layout de; setxkbmap -layout us
# properly clean the xterm
eval $(resize)
# dump the contents of an X window
xwd >window_dump.xwd; convert window_dump.xwd screenshot.png
# using xargs and get hold of the supplied argument
ls | xargs -i /bin/sh -c "ux2dos ./{} > ../dos/{}"
# show all special characters with cat
cat -vet file
# manipulate the date string of gnu ls
ls -l --time-style="+%Y%m%d"
# count characters on one line
echo "A|B|C|D|E|F" | tr -dc "|" | wc -c
# unpack rpms
rpm2cpio paket.rpm | cpio -i -d
# let man display which man page it would use
man -w lftp
# forcibly turn off the monitor via dpms with xset
xset dpms force off
# read bash.org or qdb.org quotes via lynx and sed
lynx -dump http://bash.org/?572797 | sed -n "/Browse/,/Browse/p"
# Seach google for a definition of a word
lynx -dump "http://www.google.com/search?hl=all&q=define:ramen" | sed -n "/Definitions of/,/Related phrases/p"
# display current system state with freebsd 4.x
systat -vmstat
# decrypt a .vob file
FindKey -s VTS_01_4.VOB | DecVOB VTS_01_4.VOB >vts.new.vob
# show full timestamp with ls
ls --full-time
# check if the terminal is a tty
[ -t 0 ] || [ -t 1 ] || echo not a tty
# delete a user account in freebsd with pw
pw userdel exim -r
# monitor the routing table in freebsd via route
route -n monitor
# how to checkout without the CVS directories
cvs export -D now ploc
# extract microsoft cabinet archives using cabextract
cabextract --lowercase --directory=cab-contents file.cab
# convert uppercase to lowercase using tr
tr '[:upper:]' '[:lower:]'
# simple counting using bash
for i in {1..200} ; do echo $i ; done
# make a screenshot from a text console under FreeBSD
vidcontrol -p shot.scr
# add line numbers to a file using nl
nl .bashrc
# multiply a matrix using bash
echo {a,b,c}{1,2,3}{X,Y,Z}
# create an account on a freebsd machine using pw
pw useradd snowy -g 100 -c 159 -c "Nicholas Stallard" -m -d /usr/users/support/snowy -s /bin/bash
# tcp/ip transfer using netcat
target: nc -l 5555 | "cd ; cpio -idm"
source: find -print | cpio -H newc -oaB | nc 5555
# generate a line of numbers using seq and gawk
seq 136 145 | gawk '$1=$1' RS=
# save the router configuration
cmd=save
cmd=put_all file=hurgel host=212.9.190.16
# macosx, find out what process writes a file
fs_usage -w -f filesys | fgrep testfile
# list files in current directory, sort by, and show, ctime (time of last modification of file status information), use a long listing format but do not list group information, print the index number of each file, reverse order while sorting and print the size of each file, in blocks
ls -clitoris
# remove a file whose name starts with ‘-’
rm -- --exclude=.svn*
rm ./--exclude\=.svn\*
# ANSI color codes
RS=“\[33[0m\]“ # reset
HC=“\[33[1m\]“ # hicolor
UL=“\[33[4m\]“ # underline
INV=“\[33[7m\]“ # inverse background and foreground
FBLK=“\[33[30m\]“ # foreground black
FRED=“\[33[31m\]“ # foreground red
FGRN=“\[33[32m\]“ # foreground green
FYEL=“\[33[33m\]“ # foreground yellow
FBLE=“\[33[34m\]“ # foreground blue
FMAG=“\[33[35m\]“ # foreground magenta
FCYN=“\[33[36m\]“ # foreground cyan
FWHT=“\[33[37m\]“ # foreground white
BBLK=“\[33[40m\]“ # background black
BRED=“\[33[41m\]“ # background red
BGRN=“\[33[42m\]“ # background green
BYEL=“\[33[43m\]“ # background yellow
BBLE=“\[33[44m\]“ # background blue
BMAG=“\[33[45m\]“ # background magenta
BCYN=“\[33[46m\]“ # background cyan
BWHT=“\[33[47m\]“ # background white
This is so you don’t have to write all that ANSI code over and over again. Now you can just use $FBLE to set foreground (text) to blue, instead of \[33[34m\].
