Elastic RequestError(400, 'mapper_parsing_exception', 'failed to parse')
Trying to parse data into elastic, The data which is giving the fault is:
{
"_index": "packets-2020-02-06",
"_type": "doc",
"_score": null,
"_source": {
"layers": {
"frame": {
"frame.interface_id": "0",
"frame.interface_id_tree": {
"frame.interface_name": "\\Device\\NPF_{914D83E8-6046-4A07-9C99-8DFFF2BDDCCB}",
"frame.interface_description": "Wi-Fi"
},
"frame.encap_type": "1",
"frame.time": "Feb 6, 2020 18:14:39.799011000 GMT Standard Time",
"frame.offset_shift": "0.000000000",
"frame.time_epoch": "1581012879.799011000",
"frame.time_delta": "0.000000000",
"frame.time_delta_displayed": "0.000000000",
"frame.time_relative": "53.034241000",
"frame.number": "3399",
"frame.len": "104",
"frame.cap_len": "104",
"frame.marked": "0",
"frame.ignored": "0",
"frame.protocols": "eth:ethertype:ip:udp:dns"
},
"eth": {
"eth.dst": "9c:b6:d0:0f:8d:07",
"eth.dst_tree": {
"eth.dst_resolved": "RivetNet_0f:8d:07",
"eth.dst.oui": "10270416",
"eth.dst.oui_resolved": "Rivet Networks",
"eth.addr": "9c:b6:d0:0f:8d:07",
"eth.addr_resolved": "RivetNet_0f:8d:07",
"eth.addr.oui": "10270416",
"eth.addr.oui_resolved": "Rivet Networks",
"eth.dst.lg": "0",
"eth.lg": "0",
"eth.dst.ig": "0",
"eth.ig": "0"
},
"eth.src": "d8:67:d9:04:03:44",
"eth.src_tree": {
"eth.src_resolved": "Cisco_04:03:44",
"eth.src.oui": "14182361",
"eth.src.oui_resolved": "Cisco Systems, Inc",
"eth.addr": "d8:67:d9:04:03:44",
"eth.addr_resolved": "Cisco_04:03:44",
"eth.addr.oui": "14182361",
"eth.addr.oui_resolved": "Cisco Systems, Inc",
"eth.src.lg": "0",
"eth.lg": "0",
"eth.src.ig": "0",
"eth.ig": "0"
},
"eth.type": "0x00000800"
},
"ip": {
"ip.version": "4",
"ip.hdr_len": "20",
"ip.dsfield": "0x00000000",
"ip.dsfield_tree": {
"ip.dsfield.dscp": "0",
"ip.dsfield.ecn": "0"
},
"ip.len": "90",
"ip.id": "0x0000e0e6",
"ip.flags": "0x00000000",
"ip.flags_tree": {
"ip.flags.rb": "0",
"ip.flags.df": "0",
"ip.flags.mf": "0"
},
"ip.frag_offset": "0",
"ip.ttl": "62",
"ip.proto": "17",
"ip.checksum": "0x00006179",
"ip.checksum.status": "2",
"ip.src": "152.78.255.72",
"ip.addr": "152.78.255.72",
"ip.src_host": "152.78.255.72",
"ip.host": "152.78.255.72",
"ip.dst": "10.14.152.142",
"ip.addr": "10.14.152.142",
"ip.dst_host": "10.14.152.142",
"ip.host": "10.14.152.142"
},
"udp": {
"udp.srcport": "53",
"udp.dstport": "49790",
"udp.port": "53",
"udp.port": "49790",
"udp.length": "70",
"udp.checksum": "0x0000a600",
"udp.checksum.status": "2",
"udp.stream": "49",
"Timestamps": {
"udp.time_relative": "1.534413000",
"udp.time_delta": "0.248346000"
}
},
"dns": {
"dns.id": "0x0000e506",
"dns.id_tree": {
"_ws.expert": {
"dns.retransmit_response": "",
"_ws.expert.message": "DNS response retransmission. Original response in frame 3174",
"_ws.expert.severity": "6291456",
"_ws.expert.group": "150994944"
}
},
"dns.flags": "0x00008180",
"dns.flags_tree": {
"dns.flags.response": "1",
"dns.flags.opcode": "0",
"dns.flags.authoritative": "0",
"dns.flags.truncated": "0",
"dns.flags.recdesired": "1",
"dns.flags.recavail": "1",
"dns.flags.z": "0",
"dns.flags.authenticated": "0",
"dns.flags.checkdisable": "0",
"dns.flags.rcode": "0"
},
"dns.count.queries": "1",
"dns.count.answers": "1",
"dns.count.auth_rr": "0",
"dns.count.add_rr": "0",
"Queries": {
"www.google.co.uk: type AAAA, class IN": {
"dns.qry.name": "www.google.co.uk",
"dns.qry.name.len": "16",
"dns.count.labels": "4",
"dns.qry.type": "28",
"dns.qry.class": "0x00000001"
}
},
"Answers": {
"www.google.co.uk: type AAAA, class IN, addr 2a00:1450:4009:800::2003": {
"dns.resp.name": "www.google.co.uk",
"dns.resp.type": "28",
"dns.resp.class": "0x00000001",
"dns.resp.ttl": "293",
"dns.resp.len": "16",
"dns.aaaa": "2a00:1450:4009:800::2003"
}
},
"dns.retransmit_response_in": "3174",
"dns.retransmission": "1"
}
}
}
},
And:
{
"_index": "packets-2020-02-06",
"_type": "doc",
"_score": null,
"_source": {
"layers": {
"frame": {
"frame.interface_id": "0",
"frame.interface_id_tree": {
"frame.interface_name": "\\Device\\NPF_{914D83E8-6046-4A07-9C99-8DFFF2BDDCCB}",
"frame.interface_description": "Wi-Fi"
},
"frame.encap_type": "1",
"frame.time": "Feb 6, 2020 18:14:39.799011000 GMT Standard Time",
"frame.offset_shift": "0.000000000",
"frame.time_epoch": "1581012879.799011000",
"frame.time_delta": "0.000000000",
"frame.time_delta_displayed": "0.000000000",
"frame.time_relative": "53.034241000",
"frame.number": "3400",
"frame.len": "92",
"frame.cap_len": "92",
"frame.marked": "0",
"frame.ignored": "0",
"frame.protocols": "eth:ethertype:ip:udp:dns"
},
"eth": {
"eth.dst": "9c:b6:d0:0f:8d:07",
"eth.dst_tree": {
"eth.dst_resolved": "RivetNet_0f:8d:07",
"eth.dst.oui": "10270416",
"eth.dst.oui_resolved": "Rivet Networks",
"eth.addr": "9c:b6:d0:0f:8d:07",
"eth.addr_resolved": "RivetNet_0f:8d:07",
"eth.addr.oui": "10270416",
"eth.addr.oui_resolved": "Rivet Networks",
"eth.dst.lg": "0",
"eth.lg": "0",
"eth.dst.ig": "0",
"eth.ig": "0"
},
"eth.src": "d8:67:d9:04:03:44",
"eth.src_tree": {
"eth.src_resolved": "Cisco_04:03:44",
"eth.src.oui": "14182361",
"eth.src.oui_resolved": "Cisco Systems, Inc",
"eth.addr": "d8:67:d9:04:03:44",
"eth.addr_resolved": "Cisco_04:03:44",
"eth.addr.oui": "14182361",
"eth.addr.oui_resolved": "Cisco Systems, Inc",
"eth.src.lg": "0",
"eth.lg": "0",
"eth.src.ig": "0",
"eth.ig": "0"
},
"eth.type": "0x00000800"
},
"ip": {
"ip.version": "4",
"ip.hdr_len": "20",
"ip.dsfield": "0x00000000",
"ip.dsfield_tree": {
"ip.dsfield.dscp": "0",
"ip.dsfield.ecn": "0"
},
"ip.len": "78",
"ip.id": "0x0000e0e7",
"ip.flags": "0x00000000",
"ip.flags_tree": {
"ip.flags.rb": "0",
"ip.flags.df": "0",
"ip.flags.mf": "0"
},
"ip.frag_offset": "0",
"ip.ttl": "62",
"ip.proto": "17",
"ip.checksum": "0x00006184",
"ip.checksum.status": "2",
"ip.src": "152.78.255.72",
"ip.addr": "152.78.255.72",
"ip.src_host": "152.78.255.72",
"ip.host": "152.78.255.72",
"ip.dst": "10.14.152.142",
"ip.addr": "10.14.152.142",
"ip.dst_host": "10.14.152.142",
"ip.host": "10.14.152.142"
},
"udp": {
"udp.srcport": "53",
"udp.dstport": "54030",
"udp.port": "53",
"udp.port": "54030",
"udp.length": "58",
"udp.checksum": "0x0000b889",
"udp.checksum.status": "2",
"udp.stream": "48",
"Timestamps": {
"udp.time_relative": "1.534593000",
"udp.time_delta": "0.248346000"
}
},
"dns": {
"dns.id": "0x0000c4af",
"dns.id_tree": {
"_ws.expert": {
"dns.retransmit_response": "",
"_ws.expert.message": "DNS response retransmission. Original response in frame 3173",
"_ws.expert.severity": "6291456",
"_ws.expert.group": "150994944"
}
},
"dns.flags": "0x00008180",
"dns.flags_tree": {
"dns.flags.response": "1",
"dns.flags.opcode": "0",
"dns.flags.authoritative": "0",
"dns.flags.truncated": "0",
"dns.flags.recdesired": "1",
"dns.flags.recavail": "1",
"dns.flags.z": "0",
"dns.flags.authenticated": "0",
"dns.flags.checkdisable": "0",
"dns.flags.rcode": "0"
},
"dns.count.queries": "1",
"dns.count.answers": "1",
"dns.count.auth_rr": "0",
"dns.count.add_rr": "0",
"Queries": {
"www.google.co.uk: type A, class IN": {
"dns.qry.name": "www.google.co.uk",
"dns.qry.name.len": "16",
"dns.count.labels": "4",
"dns.qry.type": "1",
"dns.qry.class": "0x00000001"
}
},
"Answers": {
"www.google.co.uk: type A, class IN, addr 216.58.204.3": {
"dns.resp.name": "www.google.co.uk",
"dns.resp.type": "1",
"dns.resp.class": "0x00000001",
"dns.resp.ttl": "219",
"dns.resp.len": "4",
"dns.a": "216.58.204.3"
}
},
"dns.retransmit_response_in": "3173",
"dns.retransmission": "1"
}
}
}
}
I have created an elastic mapping which is set to dynamic:false
The mapping is:
{
"dynamic": "false",
"properties": {
"tcp": {
"properties": {
"Timestamps": {
"properties": {
"tcp.time_delta": {
"type": "float"
},
"tcp.time_relative": {
"type": "float"
}
}
},
"tcp.seq": {
"type": "integer"
},
"tcp.srcport": {
"type": "integer"
},
"tcp.nxtseq": {
"type": "integer"
},
"tcp.flags": {
"type": "text"
},
"tcp.options": {
"type": "text"
},
"tcp.stream": {
"type": "integer"
},
"tcp.dstport": {
"type": "integer"
},
"tcp.len": {
"type": "integer"
},
"tcp.ack": {
"type": "integer"
}
}
},
"udp": {
"properties": {
"Timestamps": {
"properties": {
"udp.time_delta": {
"type": "float"
},
"udp.time_relative": {
"type": "float"
}
}
},
"udp.srcport": {
"type": "long"
},
"udp.dstport": {
"type": "long"
},
"udp.stream": {
"type": "integer"
},
"udp.length": {
"type": "integer"
}
}
},
"ipv6": {
"properties": {
"ipv6.tclass": {
"type": "text"
},
"ipv6.flow": {
"type": "text"
},
"ip.version": {
"type": "keyword"
},
"ipv6.src": {
"type": "ip"
},
"ipv6.version": {
"type": "keyword"
},
"ipv6.dst": {
"type": "ip"
}
}
},
"ip": {
"properties": {
"ip.hdr_len": {
"type": "integer"
},
"ip.src": {
"type": "ip"
},
"ip.frag_offset": {
"type": "integer"
},
"ip.len": {
"type": "integer"
},
"ip.dst": {
"type": "ip"
},
"ip.proto": {
"type": "integer"
},
"ip.ttl": {
"type": "integer"
},
"ip.id": {
"type": "text"
},
"ip.version": {
"type": "integer"
}
}
},
"eth": {
"properties": {
"eth.dst": {
"type": "keyword"
},
"eth.src": {
"type": "keyword"
}
}
},
"dns": {
"properties": {
"dns.id": {
"type": "text"
},
"dns.count.auth_rr": {
"type": "integer"
},
"dns.count.answers": {
"type": "integer"
},
"dns.count.queries": {
"type": "integer"
}
}
},
"tls": {
"properties": {
"tls.record": {
"properties": {
"tls.record.length": {
"type": "integer"
},
"tls.app_data": {
"type": "text"
},
"tls.record.version": {
"type": "text"
},
"tls.record.content_type": {
"type": "keyword"
}
}
}
}
},
"file.origin": {
"type": "text"
},
"arp": {
"properties": {
"arp.hw.type": {
"type": "keyword"
},
"arp.src.proto_ipv4": {
"type": "ip"
},
"arp.dst.proto_ipv4": {
"type": "ip"
},
"arp.proto.type": {
"type": "text"
},
"arp.src.hw_mac": {
"type": "text"
},
"arp.hw.size": {
"type": "integer"
},
"arp.proto.size": {
"type": "integer"
},
"arp.dst.hw_mac": {
"type": "text"
},
"arp.opcode": {
"type": "keyword"
}
}
},
"frame": {
"properties": {
"frame.time": {
"type": "text"
},
"frame.interface_id": {
"type": "integer"
},
"frame.protocols": {
"type": "keyword"
},
"frame.len": {
"type": "long"
},
"frame.time_epoch": {
"format": "epoch_second",
"type": "date"
},
"frame.interface_id_tree": {
"properties": {
"frame.interface_name": {
"type": "keyword"
},
"frame.interface_description": {
"type": "keyword"
}
}
},
"frame.number": {
"type": "long"
}
}
}
}
}
I am not sure what is causing the error, tried to make sure all the fields set in the mapping are set to the correct values which are seen in the data trying to parse.
Any help ?
asked on Stack Overflow Feb 8, 2020 by 
O. J.
O. J.0 Answers
Nobody has answered this question yet.
User contributions licensed under CC BY-SA 3.0