Bad Data 技術手冊

正規化文字

|

61

for non-ASCII code points

"""

counts = defaultdict(int)

for c in s:

if ord(c) > 127:

counts[ord(c)] += 1

return counts

>>> def stream_targeted_non_ascii_snippets(s,

... target_byte, n_before=15, n_after=15):

"""

s is a byte string possibly containing non-ascii

characters

target_byte is code point

n_before and n_after specify a window size

this function is a generator for snippets

containing the n_before bytes before

target_byte, target_byte itself, and the n_after

bytes that follow it.

"""

for idx, c in enumerate(s):

if ord(c) == target_byte:

start = max(idx - n_before, 0)

end = idx + n_after + 1

yield(s[start:end])

>>> sorted(get_non_ascii_byte_counts(s).items(), ...