12 Essential Python Tricks Every Developer Should Know

Python programming commonly uses the following twelve basic techniques: regular‑expression replacement, directory traversal, list sorting, list deduplication, dictionary sorting, conversion among dictionaries, lists and strings, datetime operations, command‑line argument parsing with getopt, formatted print output, numeral base conversion, invoking system commands or scripts, and file read/write.

1. Regular‑Expression Replacement

Goal: replace the substring overview.gif in a string line with another string.

>> line = '<IMG ALIGN="middle" SRC=\'#\''
>>> mo = re.compile(r'(?<=SRC)"([\w+\.]+)"', re.I)
>>> mo.sub(r'"\1****"', line)
'<IMG ALIGN="middle" SRC=\'#\''
>>> mo.sub(r'replace_str_\1', line)
'<IMG ALIGN="middle" replace_str_overview.gif BORDER="0" ALT="">
>>> mo.sub(r'"testetstset"', line)
'<IMG ALIGN="middle" SRC=\'#\''

Note: \1 refers to the captured group and can be used directly in the replacement string.

2. Directory Traversal

Use os.walk to iterate over a directory and collect specific files.

import os
fileList = []
rootdir = "/data"
for root, subFolders, files in os.walk(rootdir):
    if '.svn' in subFolders:
        subFolders.remove('.svn')  # exclude specific directories
    for file in files:
        if file.find(".t2t") != -1:  # find files with a particular extension
            file_dir_path = os.path.join(root, file)
            fileList.append(file_dir_path)
print fileList

3. List Sorting by Column

Sort a list of tuples based on a specific column, optionally in reverse order.

>> a = [('2011-03-17', '2.26', 6429600, '0.0'),
...      ('2011-03-16', '2.26', 12036900, '-3.0'),
...      ('2011-03-15', '2.33', 15615500, '-19.1')]
>>> b = sorted(a, key=lambda result: result[1], reverse=True)
>>> print b
[('2011-03-15', '2.33', 15615500, '-19.1'), ('2011-03-17', '2.26', 6429600, '0.0'), ('2011-03-16', '2.26', 12036900, '-3.0')]
>>> c = sorted(a, key=lambda result: result[2], reverse=True)
>>> print c
[('2011-03-15', '2.33', 15615500, '-19.1'), ('2011-03-16', '2.26', 12036900, '-3.0'), ('2011-03-17', '2.26', 6429600, '0.0')]

4. List Deduplication

Remove duplicate elements from a list using set.

>> lst = [(1,'sss'), (2,'fsdf'), (1,'sss'), (3,'fd')]
>>> set(lst)
set([(2, 'fsdf'), (3, 'fd'), (1, 'sss')])
>>> lst = [1, 1, 3, 4, 4, 5, 6, 7, 6]
>>> set(lst)
set([1, 3, 4, 5, 6, 7])

5. Dictionary Sorting

Sort a dictionary by its values using operator.itemgetter.

>> from operator import itemgetter
>>> aa = {"a":"1", "sss":"2", "ffdf":"5", "ffff2":"3"}
>>> sort_aa = sorted(aa.items(), key=itemgetter(1))
>>> print sort_aa
[('a', '1'), ('sss', '2'), ('ffff2', '3'), ('ffdf', '5')]

6. Conversions Between Dict, List and String

Generate a database connection string from a dictionary and convert a string back to a dictionary.

>> params = {"server":"mpilgrim", "database":"master", "uid":"sa", "pwd":"secret"}
>>> ["%s=%s" % (k, v) for k, v in params.items()]
['server=mpilgrim', 'uid=sa', 'database=master', 'pwd=secret']
>>> ";".join(["%s=%s" % (k, v) for k, v in params.items()])
'server=mpilgrim;uid=sa;database=master;pwd=secret'
>>> a = 'server=mpilgrim;uid=sa;database=master;pwd=secret'
>>> aa = {}
>>> for i in a.split(';'):
...     aa[i.split('=', 1)[0]] = i.split('=', 1)[1]
>>> print aa
{'pwd': 'secret', 'database': 'master', 'uid': 'sa', 'server': 'mpilgrim'}

7. Datetime Operations

Convert datetime objects to strings, compare times, compute time differences, and parse strings into datetime objects.

>> import datetime
>>> datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
'2011-01-20 14:05'
>>> import time
>>> t1 = time.strptime('2011-01-20 14:05', "%Y-%m-%d %H:%M")
>>> t2 = time.strptime('2011-01-20 16:05', "%Y-%m-%d %H:%M")
>>> t1 > t2
False
>>> t1 < t2
True
>>> datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
'2011-01-20 15:02'
>>> (datetime.datetime.now() - datetime.timedelta(hours=8)).strftime("%Y-%m-%d %H:%M")
'2011-01-20 07:03'
>>> endtime = datetime.datetime.strptime('20100701', "%Y%m%d")
>>> type(endtime)
<type 'datetime.datetime'>
>>> print endtime
2010-07-01 00:00:00
>>> a = 1302153828
>>> time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(a))
'2011-04-07 13:23:48'

8. Command‑Line Argument Parsing (getopt)

Parse script options using the getopt module.

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os, getopt

def usage():
    print '''
Usage: analyse_stock.py [options...]
Options:
  -e : Exchange Name
  -c : User‑Defined Category Name
  -f : Read stock info from file and save to db
  -d : delete from db by stock code
  -n : stock name
  -s : stock code
  -h : this help info
'''
    sys.exit()

opts, args = getopt.getopt(sys.argv[1:], 'he:c:f:d:n:s:')
for opt, arg in opts:
    if opt in ('-h', '--help'):
        usage()
    elif opt == '-d':
        print "del stock %s" % arg
    elif opt == '-f':
        print "read file %s" % arg
    elif opt == '-c':
        print "user‑defined %s " % arg
    elif opt == '-e':
        print "Exchange Name %s" % arg
    elif opt == '-s':
        print "Stock code %s" % arg
    elif opt == '-n':
        print "Stock name %s" % arg

9. Formatted Print Output

Various ways to format strings, numbers and floating‑point values.

# Truncate string to first 3 characters
str = "abcdefg"
print "%.3s" % str   # abc
# Fixed width output (width 10)
print "%10s" % str   #      abcdefg
# Width with precision
print "%10.3s" % str #       abc
# Floating‑point formatting using fpformat
import fpformat
a = 0.0030000000005
b = fpformat.fix(a, 6)
print b               # 0.003000
# Rounding with Decimal
from decimal import *
a = Decimal('2.26')
b = Decimal('2.29')
c = a - b
print c               # -0.03
print c * 100          # -3.0

9.2 Numeral Base Conversion

Convert an integer to hexadecimal, decimal and octal representations.

>> num = 10
>>> print "Hex = %x, Dec = %d, Oct = %o" % (num, num, num)
Hex = a, Dec = 10, Oct = 12

10. Invoking System Commands or Scripts

Execute shell commands from Python using os.system, os.popen and commands.getstatusoutput.

>> import os
>>> os.system('ls -l /proc/cpuinfo')
0
>>> out = os.popen('ls -l /proc/cpuinfo')
>>> print out.read()
-r--r--r-- 1 root root 0 Mar 29 16:59 /proc/cpuinfo
>>> import commands
>>> commands.getstatusoutput('ls /bin/ls')
(0, '/bin/ls')

11. Capturing Ctrl+C and Ctrl+D Events

Handle keyboard interrupts and EOF errors gracefully.

try:
    do_some_func()
except KeyboardInterrupt:
    print "User Press Ctrl+C, Exit"
except EOFError:
    print "User Press Ctrl+D, Exit"

12. File Read/Write

Read an entire file into a list, read line‑by‑line, and write data using write or writelines.

# Read whole file (fast, small files)
track_file = "track_stock.conf"
fd = open(track_file)
content_list = fd.readlines()
fd.close()
for line in content_list:
    print line
# Read line‑by‑line (slow, large files)
fd = open(file_path)
fd.seek(0)
title = fd.readline()
keyword = fd.readline()
uuid = fd.readline()
fd.close()
# Write examples
fd.write(str)          # writes string without newline
fd.writelines(content) # writes each element of content as‑is

The above snippets constitute a concise reference of twelve frequently used Python basics for developers.