PyRDP format #959
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -110,6 +110,7 @@ | |
| |[`protobuf`](#protobuf) |Protobuf |<sub></sub>| | ||
| |`protobuf_widevine` |Widevine protobuf |<sub>`protobuf`</sub>| | ||
| |`pssh_playready` |PlayReady PSSH |<sub></sub>| | ||
| |[`pyrdp`](#pyrdp) |PyRDP Replay Files |<sub></sub>| | ||
| |[`rtmp`](#rtmp) |Real-Time Messaging Protocol |<sub>`amf0` `mpeg_asc`</sub>| | ||
| |`sll2_packet` |Linux cooked capture encapsulation v2 |<sub>`inet_packet`</sub>| | ||
| |`sll_packet` |Linux cooked capture encapsulation |<sub>`inet_packet`</sub>| | ||
|
|
@@ -1195,6 +1196,16 @@ $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file | |
| ### References | ||
| - https://developers.google.com/protocol-buffers/docs/encoding | ||
|
|
||
| ## pyrdp | ||
| PyRDP Replay Files. | ||
|
|
||
| ### Authors | ||
| - Olivier Bilodeau <[email protected]>, Maintainer | ||
| - Lisandro Ubiedo, Author | ||
|
|
||
| ### References | ||
| - https://github.com/GoSecure/pyrdp | ||
|
|
||
| ## rtmp | ||
| Real-Time Messaging Protocol. | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| // Copyright (c) 2022-2023 GoSecure Inc. | ||
| // Copyright (c) 2024 Flare Systems | ||
| // Licensed under the MIT License | ||
| package pdu | ||
|
|
||
| import ( | ||
| "github.com/wader/fq/pkg/decode" | ||
| "github.com/wader/fq/pkg/scalar" | ||
| ) | ||
|
|
||
| const ( | ||
| RDP4 = 0x80001 | ||
| RDP5 = 0x80004 | ||
| RDP10 = 0x80005 | ||
| RDP10_1 = 0x80006 | ||
| RDP10_2 = 0x80007 | ||
| RDP10_3 = 0x80008 | ||
| RDP10_4 = 0x80009 | ||
| RDP10_5 = 0x8000a | ||
| RDP10_6 = 0x8000b | ||
| RDP10_7 = 0x8000c | ||
| RDP10_8 = 0x8000d | ||
| RDP10_9 = 0x8000e | ||
| RDP10_10 = 0x8000f | ||
| ) | ||
|
|
||
| var RDPVersionMap = scalar.UintMapSymStr{ | ||
| RDP4: "4", | ||
| RDP5: "5", | ||
| RDP10: "10", | ||
| RDP10_1: "10_1", | ||
| RDP10_2: "10_2", | ||
| RDP10_3: "10_3", | ||
| RDP10_4: "10_4", | ||
| RDP10_5: "10_5", | ||
| RDP10_6: "10_6", | ||
| RDP10_7: "10_7", | ||
| RDP10_8: "10_8", | ||
| RDP10_9: "10_9", | ||
| RDP10_10: "10_10", | ||
| } | ||
|
|
||
| const ( | ||
| CLIENT_CORE = 0xc001 | ||
| CLIENT_SECURITY = 0xc002 | ||
| CLIENT_NETWORK = 0xc003 | ||
| CLIENT_CLUSTER = 0xc004 | ||
| ) | ||
|
|
||
| var clientDataMap = scalar.UintMapSymStr{ | ||
| CLIENT_CORE: "core", | ||
| CLIENT_SECURITY: "security", | ||
| CLIENT_NETWORK: "network", | ||
| CLIENT_CLUSTER: "cluster", | ||
| } | ||
|
|
||
| func parseClientData(d *decode.D, length int64) { | ||
| d.FieldStruct("client_data", func(d *decode.D) { | ||
| header := d.FieldU16("header", clientDataMap) | ||
| dataLen := int64(d.FieldU16("length") - 4) | ||
|
|
||
| switch header { | ||
| case CLIENT_CORE: | ||
| ParseClientDataCore(d, dataLen) | ||
| case CLIENT_SECURITY: | ||
| ParseClientDataSecurity(d, dataLen) | ||
| case CLIENT_NETWORK: | ||
| ParseClientDataNetwork(d, dataLen) | ||
| case CLIENT_CLUSTER: | ||
| ParseClientDataCluster(d, dataLen) | ||
| default: | ||
| // Assert() once all functions are implemented and tested. | ||
| d.FieldRawLen("data", dataLen*8) | ||
| return | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| func ParseClientDataCore(d *decode.D, length int64) { | ||
| d.FieldU32("version", RDPVersionMap) | ||
| d.FieldU16("desktop_width") | ||
| d.FieldU16("desktop_height") | ||
| d.FieldU16("color_depth") | ||
| d.FieldU16("sas_sequence") | ||
| d.FieldU32("keyboard_layout") | ||
| d.FieldU32("client_build") | ||
| d.FieldUTF16LE("client_name", 32, scalar.StrActualTrim("\x00")) | ||
| d.FieldU32("keyboard_type") | ||
| d.FieldU32("keyboard_sub_type") | ||
| d.FieldU32("keyboard_function_key") | ||
| d.FieldRawLen("ime_file_name", 64*8) | ||
| d.FieldRawLen("code_data", 98*8) | ||
| } | ||
|
|
||
| func ParseClientDataSecurity(d *decode.D, length int64) { | ||
| d.FieldU32("encryption_methods") | ||
| d.FieldU32("ext_encryption_methods") | ||
| } | ||
|
|
||
| func ParseClientDataNetwork(d *decode.D, length int64) { | ||
| d.FieldU32("channel_count") | ||
| length -= 4 | ||
| d.FieldRawLen("channel_def_array", length*8) | ||
| } | ||
|
|
||
| func ParseClientDataCluster(d *decode.D, length int64) { | ||
| d.FieldU32("flags") | ||
| d.FieldU32("redirected_session_id") | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| // Copyright (c) 2022-2023 GoSecure Inc. | ||
| // Copyright (c) 2024 Flare Systems | ||
| // Licensed under the MIT License | ||
| package pdu | ||
|
|
||
| import ( | ||
| "github.com/wader/fq/pkg/decode" | ||
| "github.com/wader/fq/pkg/scalar" | ||
| ) | ||
|
|
||
| func parseClientInfo(d *decode.D, length int64) { | ||
| d.FieldStruct("client_info", func(d *decode.D) { | ||
| pos := d.Pos() | ||
| var ( | ||
| isUnicode bool | ||
| hasNull bool | ||
| nullN uint64 = 0 | ||
| unicodeN uint64 = 0 | ||
| ) | ||
| codePage := d.FieldU32("code_page") | ||
| d.FieldStruct("flags", func(d *decode.D) { | ||
| d.FieldBool("compression") | ||
| d.FieldBool("logonnotify") | ||
| d.FieldBool("maximizeshell") | ||
| isUnicode = d.FieldBool("unicode") | ||
| d.FieldBool("autologon") | ||
| d.FieldRawLen("unused0", 1) | ||
| d.FieldBool("disabledctrlaltdel") | ||
| d.FieldBool("mouse") | ||
|
|
||
| d.FieldBool("rail") | ||
| d.FieldBool("force_encrypted_cs_pdu") | ||
| d.FieldBool("remoteconsoleaudio") | ||
| d.FieldRawLen("unused1", 4) | ||
| d.FieldBool("enablewindowskey") | ||
|
|
||
| d.FieldBool("reserved1") | ||
| d.FieldBool("video_disable") | ||
| d.FieldBool("audiocapture") | ||
| d.FieldBool("using_saved_creds") | ||
| d.FieldBool("noaudioplayback") | ||
| d.FieldBool("password_is_sc_pin") | ||
| d.FieldBool("mouse_has_wheel") | ||
| d.FieldBool("logonerrors") | ||
|
|
||
| d.FieldRawLen("unused2", 6) | ||
| d.FieldBool("hidef_rail_supported") | ||
| d.FieldBool("reserved2") | ||
| }) | ||
|
|
||
| hasNull = (codePage == 1252 || isUnicode) | ||
|
|
||
| if hasNull { | ||
| nullN = 1 | ||
| } | ||
| if isUnicode { | ||
| unicodeN = 2 | ||
| } | ||
|
|
||
| domainLength := int(d.FieldU16("domain_length") + nullN*unicodeN) | ||
| usernameLength := int(d.FieldU16("username_length") + nullN*unicodeN) | ||
| passwordLength := int(d.FieldU16("password_length") + nullN*unicodeN) | ||
| alternateShellLength := int(d.FieldU16("alternate_shell_length") + nullN*unicodeN) | ||
| workingDirLength := int(d.FieldU16("working_dir_length") + nullN*unicodeN) | ||
|
|
||
| d.FieldUTF16LE("domain", domainLength, scalar.StrActualTrim("\x00")) | ||
| d.FieldUTF16LE("username", usernameLength, scalar.StrActualTrim("\x00")) | ||
| d.FieldUTF16LE("password", passwordLength, scalar.StrActualTrim("\x00")) | ||
| d.FieldUTF16LE("alternate_shell", alternateShellLength, scalar.StrActualTrim("\x00")) | ||
| d.FieldUTF16LE("working_dir", workingDirLength, scalar.StrActualTrim("\x00")) | ||
|
|
||
| extraLength := length - ((d.Pos() - pos) / 8) | ||
| if extraLength > 0 { | ||
| d.FieldStruct("extra_info", func(d *decode.D) { | ||
| d.FieldU16("address_family", scalar.UintHex) | ||
| addressLength := int(d.FieldU16("address_length")) | ||
| d.FieldUTF16LE("address", addressLength, scalar.StrActualTrim("\x00")) | ||
| clientDirLength := int(d.FieldU16("client_dir_length")) | ||
| d.FieldUTF16LE("client_dir", clientDirLength, scalar.StrActualTrim("\x00")) | ||
| // TS_TIME_ZONE_INFORMATION structure | ||
| // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpbcgr/526ed635-d7a9-4d3c-bbe1-4e3fb17585f4 | ||
| d.FieldU32("timezone_bias") | ||
| d.FieldUTF16LE("timezone_standardname", 64, scalar.StrActualTrim("\x00")) | ||
| }) | ||
|
|
||
| // XXX: there's more extra info but here's everything we need from the | ||
| // client (other than UTC info) | ||
| } | ||
| }) | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| // Copyright (c) 2022-2023 GoSecure Inc. | ||
| // Copyright (c) 2024 Flare Systems | ||
| // Licensed under the MIT License | ||
| package pdu | ||
|
|
||
| import ( | ||
| "github.com/wader/fq/pkg/decode" | ||
| "github.com/wader/fq/pkg/scalar" | ||
| ) | ||
|
|
||
| const ( | ||
| // Message types. | ||
| CB_TYPE_MONITOR_READY = 0x0001 | ||
| CB_TYPE_FORMAT_LIST = 0x0002 | ||
| CB_TYPE_FORMAT_LIST_RESPONSE = 0x0003 | ||
| CB_TYPE_FORMAT_DATA_REQUEST = 0x0004 | ||
| CB_TYPE_FORMAT_DATA_RESPONSE = 0x0005 | ||
| CB_TYPE_TEMP_DIRECTORY = 0x0006 | ||
| CB_TYPE_CLIP_CAPS = 0x0007 | ||
| CB_TYPE_FILECONTENTS_REQUEST = 0x0008 | ||
| CB_TYPE_FILECONTENTS_RESPONSE = 0x0009 | ||
| CB_TYPE_LOCK_CLIPDATA = 0x000a | ||
| CB_TYPE_UNLOCK_CLIPDATA = 0x000b | ||
| ) | ||
|
|
||
| var cbTypesMap = scalar.UintMapSymStr{ | ||
| CB_TYPE_MONITOR_READY: "monitor_ready", | ||
| CB_TYPE_FORMAT_LIST: "format_list", | ||
| CB_TYPE_FORMAT_LIST_RESPONSE: "format_list_response", | ||
| CB_TYPE_FORMAT_DATA_REQUEST: "format_data_request", | ||
| CB_TYPE_FORMAT_DATA_RESPONSE: "format_data_response", | ||
| CB_TYPE_TEMP_DIRECTORY: "temp_directory", | ||
| CB_TYPE_CLIP_CAPS: "clip_caps", | ||
| CB_TYPE_FILECONTENTS_REQUEST: "filecontents_request", | ||
| CB_TYPE_FILECONTENTS_RESPONSE: "filecontents_response", | ||
| CB_TYPE_LOCK_CLIPDATA: "lock_clipdata", | ||
| CB_TYPE_UNLOCK_CLIPDATA: "unlock_clipdata", | ||
| } | ||
|
|
||
| const ( | ||
| // Message flags. | ||
|
There was a problem hiding this comment. Could multiple bit be set? cbFlagsMap will atm only map if one bit is set There was a problem hiding this comment. I believe multiple bits could be set, yes.
It's vague and I understand the original mistake since two out of three messages are OK or FAIL which don't seem compatible. I'll fix it. I think I have a good idea for an approach. There was a problem hiding this comment. So my initial approach to achieve this was to replicate what you did in 69ec44a. Naively I wrote this: func parseClipboardData(d *decode.D, length int64) {
d.FieldStruct("clipboard_data", func(d *decode.D) {
msgType := uint16(d.FieldU16("msg_type", cbTypesMap))
d.FieldStruct("msg_flags", func(d *decode.D) {
d.FieldBool("cb_response_ok")
d.FieldBool("cb_response_fail")
d.FieldBool("cb_ascii_names")
})
dataLength := d.FieldU32("data_len")This is exactly how the protocol is documented and the
Enhancing the protocols' parsers would be straightforward. However, I realized that So to work the parser now looks like this: func parseClipboardData(d *decode.D, length int64) {
d.FieldStruct("clipboard_data", func(d *decode.D) {
msgType := uint16(d.FieldU16("msg_type", cbTypesMap))
d.FieldStruct("msg_flags", func(d *decode.D) {
d.FieldRawLen("unused0", 5)
d.FieldBool("cb_ascii_names")
d.FieldBool("cb_response_fail")
d.FieldBool("cb_response_ok")
d.FieldRawLen("unused1", 8)
})
dataLength := d.FieldU32("data_len")In addition to being harder to write, it also has a UX impact with the unnecessary unused fields being exposed to the user:
This is not how the protocol is described on the wire. Now, trying to improve this, is there a way we could make FieldStruct or a variant of it byte order aware? func parseClipboardData(d *decode.D, length int64) {
d.FieldStruct("clipboard_data", func(d *decode.D) {
msgType := uint16(d.FieldU16("msg_type", cbTypesMap))
d.FieldU16Bitfield("msg_flags", func(d *decode.D) {
d.FieldBool("cb_response_ok")
d.FieldBool("cb_response_fail")
d.FieldBool("cb_ascii_names")
})
dataLength := d.FieldU32("data_len")Using If this is too complicated to implement, I think the previous way (const list + Otherwise, is there something else in the decode API I don't see or understand that would be better? There was a problem hiding this comment.
Yeap this gets a bit messy at times with how fq was designed to work, that is to not hide anything and also give "access" to all bits somehow even unused and unknown stuff. So in general i've usually opted to let format decoders decode things as detailed as possible to have as few assumption what a end user would like to query and look at, e.g. maybe someone whats to extract or query the unused bits?
All decoding in fq currently is on purpose on bit level made to "hide" byte boundaries, this is to allow decoding bitstream formats, common for media codecs etc, without much fuzz. And as you have noticed this put some burden on some format decoders to byte align and maybe have to deal with padding, unknown or unused bits that a byte-oriented decoder would not care about, fq even automatically adds "gap" fields for bit ranges a decode skips or for some reason can't know about (trailing bits in a format with explicit ending), this is so that no bits are hidden/unreachable. So in general if there is a clear bit field it's probably best to decode it they way you did above: a struct with bool/raw fields for each bit even unused or unknown ones. Will be a bit verbose but now all of them are accessible via a jq query. Btw if there bit flag combinations etc that have some special meaning a decode can add "synthetic" fields of any type that will not be "backed" by a bit range but will be visible and queryable (ex samples count in the mp3 decoder https://github.com/wader/fq/blob/master/format/mpeg/mp3_frame.go#L195) But yeap all this becomes a bit messy and uninvited when decoding little endian bit fields that are > 1 byte :( currently there is no helpers for dealing with it but i have tried to come up with some way of specifying things in the same order as in a spec etc and the let fq figure things out. Actually in the kaitai prototype there is some support for what kaitai calls "bit-endian" that does more or less this. Sadly things get even messier when decoding some bit range inside a little endian integer as now a range a field can "span" two or more bytes and be non-continuous in the bitstream :(. Not sure how to visualise that in the "dump" tree, atm i think i would just opt to let the fiend be the range of the first and the last bit backing the field. Sorry for wall of text! and hope i managed to explain myself properly and all this is a bit of an unsolved issue with fq so i'm glad someone has input on it. There was a problem hiding this comment. You have made this definitely clearer. Having access to bit-sequence unused at the time the parser was written but that could be used later is a strong argument too. User-friendly before plugin programmer-friendly. |
||
| CB_FLAG_NONE = 0 | ||
| CB_FLAG_RESPONSE_OK = 0x0001 | ||
| CB_FLAG_RESPONSE_FAIL = 0x0002 | ||
| CB_FLAG_ASCII_NAMES = 0x0004 | ||
| ) | ||
|
|
||
| var cbFlagsMap = scalar.UintMapSymStr{ | ||
| CB_FLAG_NONE: "none", | ||
| CB_FLAG_RESPONSE_OK: "response_ok", | ||
| CB_FLAG_RESPONSE_FAIL: "response_fail", | ||
| CB_FLAG_ASCII_NAMES: "ascii_names", | ||
| } | ||
|
|
||
| var cbParseFnMap = map[uint16]interface{}{ | ||
| CB_TYPE_FORMAT_DATA_RESPONSE: parseCbFormatDataResponse, | ||
| } | ||
|
|
||
| func parseClipboardData(d *decode.D, length int64) { | ||
| d.FieldStruct("clipboard_data", func(d *decode.D) { | ||
| msgType := uint16(d.FieldU16("msg_type", cbTypesMap)) | ||
| d.FieldU16("msg_flags", cbFlagsMap) | ||
| dataLength := d.FieldU32("data_len") | ||
|
|
||
| cbParser, ok := cbParseFnMap[msgType] | ||
| if ok { | ||
| parseFn, ok := cbParser.(func(d *decode.D, length uint64)) | ||
| if ok { | ||
| parseFn(d, dataLength) | ||
| return | ||
| } | ||
| } | ||
| // Assert() once all functions are implemented. | ||
| d.FieldRawLen("data", int64(dataLength*8)) | ||
| }) | ||
| } | ||
|
|
||
| func parseCbFormatDataResponse(d *decode.D, length uint64) { | ||
| d.FieldRawLen("data", int64(length*8)) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could you verify the flags look more correct now in test.fqtest? now unicode flags is true at least and i hope i got the other ones correct
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Right now I'm not focused on the correctness of the flags but on how a protocol (or filetype) analyst looks at them.
Having a different ordering makes it confusing. Ideally it should be aligned. Check my previous comment on different approaches that could be taken. Please advise, I'm willing to help.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Microsoft documentation about the client_info flags is here: https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpbcgr/732394f5-e2b5-4ac5-8a0a-35345386b0d1
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The comment above i hope explains why this looks a bit weird in fq: fields are in "bitstream"-order which might not be how spec define them and things gets even more weird with multi-byte little endian bitfields. I wonder how fq would visualise things if it would allow a format decoder to "force" some specific field order for structs? maybe something to indicate that things are now not in bitstream order?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, I understand more the context now. I'm wondering how Wireshark does it?...
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So Wireshark does it like this:
The boolean flags in the field are all individually defined, each by a struct with a mask value provided. The container of the flags is also defined (
foo.flagsandfoo.flags.flag1both exist) and in its structure, the endianness is provided. These are static declarations.Then the dissector refers to these declarations in the order that they should be presented. It's quite verbose and there's a lot of duplication of effort but at the same time more control.
Ref: https://www.wireshark.org/docs/wsdg_html_chunked/ChDissectAdd.html and https://gitlab.com/wireshark/wireshark/-/blob/master/epan/dissectors/packet-rdp.c
That approach is not interesting but it gave me an idea. I feel like making the
FieldStuct()function endian-aware could be an elegant compromise. To avoid breaking existing code, it could be a new function calledFieldStructEndianAware()or something. If so then the fields would be properly sorted in the output.Write this:
with
FieldStructEndianAware()relying on the globald.Endianvalue.Instead of this:
It would solve many problems:
Is that something that would be easily feasible?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I tried to implement my proposal today. I looked at the
decode.FieldStructcode and realized that everything is delegated tofieldDecoder... Maybe some state should be stored in the Compound struct? An additional member used to track that this struct requires byte endianness awareness? Then the AddChild could do things differently but I don't know how to keep it elegant.It's too deep of a change for me. I give up. I'll verify the flags and use the current decoding infrastructure.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hey! thanks for researching and digging into this! sorry for slow reply. I'm on a 2 week bicycle vacation but home soon again, will look closer at this then.
But i can dump some info how i started look into this for the kaitai prototype that will require "bit-endian" which i think i related/same as this? see https://doc.kaitai.io/user_guide.html#_bit_sized_integers (search for bit-endian) for kaitais definition of it.
So in the kaitai struct prototype branch the decode context looks like this to keep som state (similar to what you proposed) https://github.com/wader/fq/blob/kaitai2/pkg/decode/decode.go#L177 and usage would that a decoder can set
d.BitEndian = decode.LittleEndianin a struct for example and the decode internals will start reading bits from bytes "in-reverse" and also keep track to fail if you try to read mixed bit-endian-ness from same byte etc. Later on other parts of the decode internals will take care of sorting the fields. Only weirdness with all of this is what to do with fields that ends up with non-continuous bit-ranges? current i'm think of just assign a bit-range of the first and last bit affecting the field.