Standard Library Gems

collections

The collections module provides specialized container types that solve problems you would otherwise solve with boilerplate. Four of them come up constantly.

Counter

Counter counts occurrences of elements in any iterable. It is a dict subclass, so missing keys return 0 instead of raising KeyError.

Python

1from collections import Counter
2
3words = ["apple", "banana", "apple", "cherry", "banana", "apple"]
4counts = Counter(words)
5
6print(counts)                # Counter({'apple': 3, 'banana': 2, 'cherry': 1})
7print(counts["apple"])       # 3
8print(counts["grape"])       # 0, missing keys return 0
9print(counts.most_common(2)) # [('apple', 3), ('banana', 2)]
10
11# Arithmetic between counters
12a = Counter(cats=3, dogs=1)
13b = Counter(cats=1, dogs=4, fish=2)
14print(a + b)   # Counter({'dogs': 5, 'cats': 4, 'fish': 2})
15print(a - b)   # Counter({'cats': 2}), negative counts are dropped

defaultdict

defaultdict takes a factory function. When you access a missing key, it calls the factory, stores the result, and returns it. No more if key not in d: d[key] = [].

Python

1from collections import defaultdict
2
3words = ["ant", "bear", "alligator", "bee", "crow", "cat"]
4
5# Group by first letter
6grouped = defaultdict(list)
7for word in words:
8    grouped[word[0]].append(word)
9
10print(dict(grouped))
11# {'a': ['ant', 'alligator'], 'b': ['bear', 'bee'], 'c': ['crow', 'cat']}
12
13# Count occurrences
14counts = defaultdict(int)
15for word in words:
16    counts[word[0]] += 1   # missing key starts at 0
17
18print(dict(counts))   # {'a': 2, 'b': 2, 'c': 2}

deque

A deque (double-ended queue) supports O(1) appends and pops from both ends. List operations on the left side are O(n) because every element shifts. Use deque for queues, breadth-first search, or sliding windows.

Python

1from collections import deque
2
3q = deque([1, 2, 3])
4q.append(4)       # add right, O(1)
5q.appendleft(0)   # add left,  O(1)  (list.insert(0, x) is O(n))
6print(q)          # deque([0, 1, 2, 3, 4])
7
8q.pop()           # remove right, O(1)
9q.popleft()       # remove left,  O(1)
10print(q)          # deque([1, 2, 3])
11
12# Fixed-size window: old items are automatically discarded
13recent = deque(maxlen=3)
14for x in range(6):
15    recent.append(x)
16print(recent)     # deque([3, 4, 5], maxlen=3)

namedtuple

namedtuple creates a tuple subclass with named fields. It is as memory-efficient as a plain tuple but as readable as an object. Use it for lightweight, immutable data containers.

Python

1from collections import namedtuple
2
3Point = namedtuple("Point", ["x", "y"])
4p = Point(3, 4)
5
6print(p.x, p.y)    # 3 4, attribute access
7print(p[0], p[1])  # 3 4, index access still works
8print(p)           # Point(x=3, y=4)
9
10x, y = p           # unpacking works too
11
12# As a return type, far clearer than returning a plain tuple
13Color = namedtuple("Color", ["red", "green", "blue"])
14
15def get_brand_color():
16    return Color(31, 173, 135)
17
18c = get_brand_color()
19print(c.green)   # 173, c[1] would be confusing

itertools

itertools is a collection of fast, memory-efficient building blocks for working with iterables. Every function returns an iterator, it generates values on demand rather than building a full list in memory first.

chain and chain.from_iterable

Concatenate any number of iterables without building intermediate lists. chain.from_iterable flattens one level of nesting.

Python

1from itertools import chain
2
3# Concatenate multiple iterables
4letters = chain("abc", "def", "ghi")
5print(list(letters))   # ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']
6
7# Flatten one level of nesting
8nested = [[1, 2], [3, 4], [5, 6]]
9flat = list(chain.from_iterable(nested))
10print(flat)   # [1, 2, 3, 4, 5, 6]

islice

Slice any iterator, not just sequences. Essential when working with generators or infinite sequences.

Python

1from itertools import islice, count
2
3# count() produces an infinite sequence: 0, 1, 2, ...
4first_ten = list(islice(count(), 10))
5print(first_ten)   # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
6
7# islice(iterable, start, stop, step)
8evens = list(islice(count(), 0, 20, 2))
9print(evens)   # [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
10
11# Read only the first 5 lines of a large file without loading all of it
12with open("large_file.txt") as f:
13    head = list(islice(f, 5))

product, combinations, permutations

Python

1from itertools import product, combinations, permutations
2
3# Cartesian product, all pairs from two sequences
4for size, color in product(["S", "M", "L"], ["red", "blue"]):
5    print(f"{size}-{color}", end="  ")
6# S-red  S-blue  M-red  M-blue  L-red  L-blue
7
8print()
9
10# All 2-element combinations (order doesn't matter, no repeats)
11print(list(combinations("ABCD", 2)))
12# [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]
13
14# All 2-element permutations (order matters)
15print(list(permutations("ABC", 2)))
16# [('A','B'), ('A','C'), ('B','A'), ('B','C'), ('C','A'), ('C','B')]

groupby

groupby groups consecutive elements by a key function. Sort the data by that key first, it only groups adjacent equal keys, not all matching keys in the sequence.

Python

1from itertools import groupby
2
3data = [
4    {"name": "Alice", "dept": "eng"},
5    {"name": "Bob",   "dept": "eng"},
6    {"name": "Carol", "dept": "hr"},
7    {"name": "Dave",  "dept": "hr"},
8    {"name": "Eve",   "dept": "eng"},
9]
10
11# Sort first, then group
12data.sort(key=lambda r: r["dept"])
13
14for dept, members in groupby(data, key=lambda r: r["dept"]):
15    names = [m["name"] for m in members]
16    print(f"{dept}: {names}")

takewhile and dropwhile

Python

1from itertools import takewhile, dropwhile
2
3nums = [2, 4, 6, 7, 8, 10]
4
5# Take elements while condition holds, stop at first failure
6evens_prefix = list(takewhile(lambda x: x % 2 == 0, nums))
7print(evens_prefix)   # [2, 4, 6]
8
9# Skip elements while condition holds, then yield the rest
10rest = list(dropwhile(lambda x: x % 2 == 0, nums))
11print(rest)           # [7, 8, 10]

functools

functools contains higher-order functions: tools that operate on or return other functions. Three of them are genuinely useful in everyday code.

lru_cache, memoization in one line

@lru_cache caches the results of a function. On subsequent calls with the same arguments, it returns the cached value instead of recomputing. Use it for pure functions with expensive or repeated calculations.

Python

1from functools import lru_cache
2
3@lru_cache(maxsize=128)
4def fibonacci(n):
5    if n < 2:
6        return n
7    return fibonacci(n - 1) + fibonacci(n - 2)
8
9print(fibonacci(35))   # 9227465
10
11# Inspect how the cache is being used
12print(fibonacci.cache_info())
13# CacheInfo(hits=33, misses=36, maxsize=128, currsize=36)
14
15# Clear the cache when needed
16fibonacci.cache_clear()

Without caching, naive recursive Fibonacci for n=35 makes over 29 million function calls. With @lru_cache, it makes 36. Arguments must be hashable, no lists or dicts.

Python 3.9 added @cache as a shorthand for @lru_cache(maxsize=None), which keeps every result forever. Use @lru_cache when you need a bounded cache; use @cache when the input space is small.

partial, pre-fill arguments

partial creates a new function with some arguments already filled in. It is a cleaner alternative to writing a one-line lambda wrapper.

Python

1from functools import partial
2
3def power(base, exponent):
4    return base ** exponent
5
6square = partial(power, exponent=2)
7cube   = partial(power, exponent=3)
8
9print(square(4))   # 16
10print(cube(3))     # 27
11
12# Common use: adapting a function for filter() or map()
13def starts_with(prefix, s):
14    return s.startswith(prefix)
15
16is_py = partial(starts_with, "py")
17files = ["pyproject.toml", "README.md", "pytest.ini", "setup.cfg"]
18print(list(filter(is_py, files)))   # ['pyproject.toml', 'pytest.ini']

reduce, fold a sequence

reduce applies a function cumulatively to a sequence, reducing it to a single value. The optional third argument is the starting value.

Python

1from functools import reduce
2import operator
3
4nums = [1, 2, 3, 4, 5]
5
6# Sum:     ((((1+2)+3)+4)+5) = 15
7print(reduce(operator.add, nums))        # 15
8
9# Product: 1*2*3*4*5 = 120
10print(reduce(operator.mul, nums))        # 120
11
12# With a starting value
13print(reduce(operator.add, nums, 100))   # 115
14
15# Flatten one level of nesting
16nested = [[1, 2], [3, 4], [5, 6]]
17flat = reduce(operator.add, nested)
18print(flat)   # [1, 2, 3, 4, 5, 6]

For simple cases like sum and product, use the built-ins sum() and math.prod() instead. reduce earns its place when the combining operation is non-trivial or you receive the function as an argument.

datetime

The datetime module provides three types you will use constantly: date (year, month, day), datetime (date plus time), and timedelta (a duration you can add or subtract).

Python

1from datetime import date, datetime, timedelta
2
3today = date.today()
4now   = datetime.now()
5
6print(today)   # 2024-03-15
7print(now)     # 2024-03-15 14:32:07.891234
8
9# Construct specific dates
10deadline = date(2024, 12, 31)
11meeting  = datetime(2024, 3, 20, 9, 30)   # March 20 at 09:30
12
13# Date arithmetic
14in_30_days = today + timedelta(days=30)
15diff = deadline - today
16print(diff.days)   # number of days until deadline

Formatting and parsing

strftime formats a datetime to a string. strptime parses a string back into a datetime. The format codes are the same in both.

Python

1from datetime import datetime
2
3dt = datetime(2024, 3, 15, 14, 32, 7)
4
5# Format: datetime → string
6print(dt.strftime("%Y-%m-%d"))             # 2024-03-15
7print(dt.strftime("%d %B %Y"))             # 15 March 2024
8print(dt.strftime("%I:%M %p"))             # 02:32 PM
9print(dt.strftime("%Y-%m-%dT%H:%M:%S"))   # 2024-03-15T14:32:07
10
11# Parse: string → datetime
12raw    = "2024-03-15 14:32:07"
13parsed = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S")
14print(type(parsed))   # <class 'datetime.datetime'>

Timezone-aware datetimes

datetime.now() returns a naive datetime with no timezone attached. For APIs, databases, and any code that crosses timezones, use timezone-aware datetimes from the start.

Python

1from datetime import datetime, timezone, timedelta
2
3# UTC, always use this for storage and APIs
4now_utc = datetime.now(timezone.utc)
5print(now_utc)   # 2024-03-15 14:32:07.123456+00:00
6
7# Convert to a specific UTC offset
8tz_plus2 = timezone(timedelta(hours=2))
9local = now_utc.astimezone(tz_plus2)
10print(local)     # 2024-03-15 16:32:07.123456+02:00
11
12# Parse an ISO 8601 string with timezone (Python 3.7+)
13ts = datetime.fromisoformat("2024-03-15T14:32:07+00:00")
14print(ts.tzinfo)   # UTC

Store datetimes in UTC. Convert to local time only at the display layer. This one rule prevents the majority of timezone-related bugs.

Common format codes

Python

1# The codes you use most often
2# %Y  four-digit year:       2024
3# %m  zero-padded month:     03
4# %d  zero-padded day:       15
5# %H  24-hour hour:          14
6# %I  12-hour hour:          02
7# %M  minute:                32
8# %S  second:                07
9# %p  AM/PM:                 PM
10# %B  full month name:       March
11# %A  full weekday name:     Friday
12# %f  microseconds:          891234
13# %z  UTC offset:            +0000

json

Python's json module converts between Python objects and JSON text. The four functions you need: loads and dumps work with strings; load and dump work with file objects.

Python

1import json
2
3# loads: JSON string -> Python object
4raw = '{"name": "Alice", "age": 30, "active": true}'
5data = json.loads(raw)
6print(data)          # {'name': 'Alice', 'age': 30, 'active': True}
7print(type(data))    # <class 'dict'>
8
9# dumps: Python object -> JSON string
10obj = {"scores": [95, 87, 91], "passed": True}
11s = json.dumps(obj)
12print(s)             # {"scores": [95, 87, 91], "passed": true}
13
14# Pretty print with indentation
15print(json.dumps(obj, indent=2))

Reading and writing files

Python

1import json
2
3config = {"host": "localhost", "port": 8080, "debug": False}
4
5# Write JSON to a file
6with open("config.json", "w", encoding="utf-8") as f:
7    json.dump(config, f, indent=2)
8
9# Read JSON from a file
10with open("config.json", encoding="utf-8") as f:
11    loaded = json.load(f)
12
13print(loaded["port"])   # 8080

Type mapping

JSON has fewer types than Python. Know what maps to what, the differences bite you when you least expect it.

Python

1import json
2
3# Python -> JSON conversions
4print(json.dumps(None))     # null
5print(json.dumps(True))     # true     (lowercase!)
6print(json.dumps(42))       # 42
7print(json.dumps(3.14))     # 3.14
8print(json.dumps("hello"))  # "hello"
9print(json.dumps([1, 2]))   # [1, 2]
10print(json.dumps({"a": 1})) # {"a": 1}
11
12# tuples become arrays; sets and custom objects fail by default
13print(json.dumps((1, 2)))   # [1, 2], tuple -> array
14
15try:
16    json.dumps({1, 2})   # set is not JSON-serializable
17except TypeError as e:
18    print(e)

Custom encoders

Subclass json.JSONEncoder to serialize types that JSON does not support natively.

Python

1import json
2import datetime
3from decimal import Decimal
4
5class AppEncoder(json.JSONEncoder):
6    def default(self, obj):
7        if isinstance(obj, datetime.date):
8            return obj.isoformat()
9        if isinstance(obj, Decimal):
10            return float(obj)
11        return super().default(obj)
12
13data = {
14    "created": datetime.date(2024, 3, 15),
15    "price":   Decimal("19.99"),
16}
17
18print(json.dumps(data, cls=AppEncoder, indent=2))