How to parse the JSON output of a running command?
Summary: I would like to parse the JSON output of tshark as it is outputted.
As of now I was parsing normal output, line by line, and each line had the complete information. It was therefore a matter of
p = subprocess.Popen("/usr/bin/tshark", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
for line in p.stdout:
event = decode_event(line)
tshark can also output pretty-printed JSON via the -T json switch (I just give the first packet, the output is a list):
[
{
"_index": "packets-2018-03-08",
"_type": "pcap_file",
"_score": null,
"_source": {
"layers": {
"frame": {
"frame.interface_id": "0",
"frame.encap_type": "1",
"frame.time": "Mar 8, 2018 16:17:20.478658037 CET",
"frame.offset_shift": "0.000000000",
"frame.time_epoch": "1520522240.478658037",
"frame.time_delta": "0.000113952",
"frame.time_delta_displayed": "0.000113952",
"frame.time_relative": "3.351515496",
"frame.number": "11133",
"frame.len": "60",
"frame.cap_len": "60",
"frame.marked": "0",
"frame.ignored": "0",
"frame.protocols": "eth:ethertype:ip:tcp"
},
"eth": {
"eth.dst": "00:50:56:bb:40:70",
"eth.dst_tree": {
"eth.dst_resolved": "Vmware_bb:40:70",
"eth.addr": "00:50:56:bb:40:70",
"eth.addr_resolved": "Vmware_bb:40:70",
"eth.lg": "0",
"eth.ig": "0"
},
"eth.src": "64:a0:e7:42:af:41",
"eth.src_tree": {
"eth.src_resolved": "Cisco_42:af:41",
"eth.addr": "64:a0:e7:42:af:41",
"eth.addr_resolved": "Cisco_42:af:41",
"eth.lg": "0",
"eth.ig": "0"
},
"eth.type": "0x00000800",
"eth.padding": "00:00:00:00:00:00"
},
"ip": {
"ip.version": "4",
"ip.hdr_len": "20",
"ip.dsfield": "0x00000000",
"ip.dsfield_tree": {
"ip.dsfield.dscp": "0",
"ip.dsfield.ecn": "0"
},
"ip.len": "40",
"ip.id": "0x00005a57",
"ip.flags": "0x00000002",
"ip.flags_tree": {
"ip.flags.rb": "0",
"ip.flags.df": "1",
"ip.flags.mf": "0"
},
"ip.frag_offset": "0",
"ip.ttl": "125",
"ip.proto": "6",
"ip.checksum": "0x0000dd25",
"ip.checksum.status": "2",
"ip.src": "10.237.78.2",
"ip.addr": "10.237.78.2",
"ip.src_host": "10.237.78.2",
"ip.host": "10.237.78.2",
"ip.dst": "10.81.99.19",
"ip.addr": "10.81.99.19",
"ip.dst_host": "10.81.99.19",
"ip.host": "10.81.99.19",
"Source GeoIP: Unknown": "",
"Destination GeoIP: Unknown": ""
},
"tcp": {
"tcp.srcport": "31316",
"tcp.dstport": "22",
"tcp.port": "31316",
"tcp.port": "22",
"tcp.stream": "0",
"tcp.len": "0",
"tcp.seq": "3025",
"tcp.ack": "774293",
"tcp.hdr_len": "20",
"tcp.flags": "0x00000010",
"tcp.flags_tree": {
"tcp.flags.res": "0",
"tcp.flags.ns": "0",
"tcp.flags.cwr": "0",
"tcp.flags.ecn": "0",
"tcp.flags.urg": "0",
"tcp.flags.ack": "1",
"tcp.flags.push": "0",
"tcp.flags.reset": "0",
"tcp.flags.syn": "0",
"tcp.flags.fin": "0",
"tcp.flags.str": "\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7A\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7"
},
"tcp.window_size_value": "2047",
"tcp.window_size": "2047",
"tcp.window_size_scalefactor": "-1",
"tcp.checksum": "0x000073f4",
"tcp.checksum.status": "2",
"tcp.urgent_pointer": "0",
"tcp.analysis": {
"tcp.analysis.acks_frame": "11126",
"tcp.analysis.ack_rtt": "0.000426928"
}
}
}
}
},
<next packet>
What would be the correct approach to parse such a stream?
When searching for stream parsing, I found a few libraries (notably NAYA), but they require a file like object.
It would seem that StringIO() would be appropriate but I do not know how to connect it with stdout?
Per @omu_negru request, specifically in case of NAYA, directly attaching stdout as in
import naya
import subprocess
def handle_message(event):
print(event)
cmd = "/usr/bin/tshark -i eth0 -T json"
proc = subprocess.Popen(cmd, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
messages = naya.stream_array(proc.stdout)
for message in messages:
handle_message(message)
raises an exception
Traceback (most recent call last):
File "/root/dev/readtshark.py", line 12, in <module>
for message in messages:
File "/usr/local/lib/python3.5/dist-packages/naya/json.py", line 544, in stream_array
token_type, token = next(token_stream)
ValueError: too many values to unpack (expected 2)
2 Answers
Actual working version
#!/usr/bin/python3
# tshark.py
import json, sys, time
output = sys.stdin
acc = '{'
def skip(output):
while True:
l = output.readline()
if l.strip() != '{':
continue
else:
break
skip(output)
print("starting")
while True:
l = output.readline()
if l.strip() != '':
acc += l.strip()
try:
o = json.loads(acc)
print(o)
skip(output)
acc = '{'
except:
pass
Launched with sudo tshark -i wlp3s0 -T json | ./tshark.py
omu_negru@omu_negru's answer gave me an idea and I ended up using the solution below.
This is basically a continuous attempt to decode a JSON and once it is decoded, it is an event I process further (here, only print)
import subprocess
import json
def handle_message(event):
print(event)
cmd = "/usr/bin/tshark -n -T json not broadcast and not multicast"
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
# skip first lines, until the [ which starts JSON
for line in proc.stdout:
if line.decode().startswith('['):
break
else:
continue
buffer = ""
for line in proc.stdout:
# remove empty and "connection" lines (a comma)
if not line.decode().strip(', \n'):
continue
buffer += line.decode('utf-8')
try:
event = json.loads(buffer)
except json.decoder.JSONDecodeError:
pass
else:
print(event)
buffer = ""
answered on Stack Overflow Mar 9, 2018 by 
WoJ
WoJUser contributions licensed under CC BY-SA 3.0