Skip to content

Commit

Permalink
Add MBR format
Browse files Browse the repository at this point in the history
  • Loading branch information
tlehman committed Jun 30, 2022
1 parent 53ca5f2 commit 467ede9
Show file tree
Hide file tree
Showing 13 changed files with 1,047 additions and 744 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ jpeg,
json,
[macho](doc/formats.md#macho),
[matroska](doc/formats.md#matroska),
[mbr](doc/formats.md#mbr),
[mp3](doc/formats.md#mp3),
mp3_frame,
[mp4](doc/formats.md#mp4),
Expand Down
4 changes: 2 additions & 2 deletions doc/dev.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
- Create a directory `format/<name>`
- Copy some similar decoder, `format/format/bson.go` is quite small, to `format/<name>/<name>.go`
- Cleanup and fill in the register struct, rename `format.BSON` and add it
to `format/fromat.go` and don't forget to change the string constant.
to `format/format.go` and don't forget to change the string constant.
- Add an import to `format/all/all.go`

### Some general tips
Expand Down Expand Up @@ -38,7 +38,7 @@ Flags can be struct with bit-fields.

`<Field>?(<reader<length>?>|<type>Fn>)(...[, scalar.Mapper...]) <type>`

- If starts with `Field` a field will be added and first argument will be name of field. If not it will just read.
- If it starts with `Field` a field will be added and first argument will be name of field. If not it will just read.
- `<reader<length>?>|<type>Fn>` a reader or a reader function
- `<reader<length>?>` Read bits using some decoder.
- `U16` unsigned 16 bit integer.
Expand Down
10 changes: 10 additions & 0 deletions doc/formats.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
|`json` |JSON |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`mbr`](#mbr) |Master&nbsp;Boot&nbsp;Record |<sub></sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`xing`</sub>|
|[`mp4`](#mp4) |ISOBMFF&nbsp;MPEG-4&nbsp;part&nbsp;12&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `flac_frame` `flac_metadatablocks` `id3v2` `image` `jpeg` `mp3_frame` `avc_au` `avc_dcr` `mpeg_es` `hevc_au` `hevc_dcr` `mpeg_pes_packet` `opus_packet` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr` `icc_profile`</sub>|
Expand Down Expand Up @@ -352,6 +353,15 @@ Return `matroska_path` string for a box decode value
- https://www.matroska.org/technical/codec_specs.html
- https://wiki.xiph.org/MatroskaOpus

### mbr

Supports decoding Master Boot Record data

#### References and links

- https://thestarman.pcministry.com/asm/mbr/PartTables.htm#mbr
- https://en.wikipedia.org/wiki/Master_boot_record

### mp3

#### Options
Expand Down
1,490 changes: 748 additions & 742 deletions doc/formats.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
1 change: 1 addition & 0 deletions format/all/all.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
_ "github.com/wader/fq/format/json"
_ "github.com/wader/fq/format/macho"
_ "github.com/wader/fq/format/matroska"
_ "github.com/wader/fq/format/mbr"
_ "github.com/wader/fq/format/mp3"
_ "github.com/wader/fq/format/mp4"
_ "github.com/wader/fq/format/mpeg"
Expand Down
11 changes: 11 additions & 0 deletions format/all/help.fqtest
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,17 @@ out https://matroska.org/technical/specs/index.html
out https://www.matroska.org/technical/basics.html
out https://www.matroska.org/technical/codec_specs.html
out https://wiki.xiph.org/MatroskaOpus
"help(mbr)"
out mbr: Master Boot Record decoder
out Supports decoding Master Boot Record data
out Examples:
out # Decode file as mbr
out $ fq -d mbr . file
out # Decode value as mbr
out ... | mbr
out References and links
out https://thestarman.pcministry.com/asm/mbr/PartTables.htm#mbr
out https://en.wikipedia.org/wiki/Master_boot_record
"help(mp3)"
out mp3: MP3 file decoder
out Options:
Expand Down
1 change: 1 addition & 0 deletions format/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ const (
JSON = "json"
MACHO = "macho"
MATROSKA = "matroska"
MBR = "mbr"
MP3 = "mp3"
MP3_FRAME = "mp3_frame"
MP4 = "mp4"
Expand Down
72 changes: 72 additions & 0 deletions format/mbr/mbr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package mbr

import (
"embed"
"fmt"

"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)

//go:embed mbr.jq
var mbrFS embed.FS

func init() {
registry.MustRegister(decode.Format{
Name: format.MBR,
Description: "Master Boot Record",
DecodeFn: mbrDecode,
Functions: []string{"_help"},
Files: mbrFS,
})
}

func decodePartitionTableEntry(d *decode.D) {
d.FieldU8("boot_indicator", scalar.UToDescription{
0x80: "active",
0x00: "inactive",
})
d.FieldStrScalarFn("starting_chs_vals", decodeCHSBytes)
d.FieldU8("partition_type", partitionTypes)
d.FieldStrScalarFn("ending_chs_vals", decodeCHSBytes)
d.FieldStrScalarFn("starting_sector", decodeCHSBytes)
d.U8() // extra byte
d.FieldScalarU32("partition_size")
}

// Because this is a fixed-sized table, I am opting to use a
// FieldStruct instead of a FieldArray
func decodePartitionTable(d *decode.D) {
d.FieldStruct("entry_1", decodePartitionTableEntry)
d.FieldStruct("entry_2", decodePartitionTableEntry)
d.FieldStruct("entry_3", decodePartitionTableEntry)
d.FieldStruct("entry_4", decodePartitionTableEntry)
}

// Source: https://thestarman.pcministry.com/asm/mbr/PartTables.htm#Decoding
func decodeCHSBytes(d *decode.D) scalar.S {
head, _ := d.Bits(8)
sectorHighBits, err := d.Bits(2)
if err != nil {
d.IOPanic(err, "chs")
}
sector, _ := d.Bits(6)
cylinderLowerBits, err := d.Bits(8)
if err != nil {
d.IOPanic(err, "chs")
}
cylinder := (sectorHighBits << 2) | cylinderLowerBits
return scalar.S{Actual: fmt.Sprintf("CHS(%x, %x, %x)", cylinder, head, sector)}
}

func mbrDecode(d *decode.D, in interface{}) interface{} {
d.Endian = decode.LittleEndian

d.FieldRawLen("code_area", 446*8)
d.FieldStruct("partition_table", decodePartitionTable)
d.FieldU16("boot_record_sig", scalar.ActualHex)
//d.AssertU(0xaa55)
return nil
}
7 changes: 7 additions & 0 deletions format/mbr/mbr.jq
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
def _mbr__help:
{ notes: "Supports decoding Master Boot Record data",
links: [
{url: "https://thestarman.pcministry.com/asm/mbr/PartTables.htm#mbr"},
{url: "https://en.wikipedia.org/wiki/Master_boot_record"}
]
};
151 changes: 151 additions & 0 deletions format/mbr/partition_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
package mbr

import (
"github.com/wader/fq/pkg/scalar"
)

// Source: https://thestarman.pcministry.com/asm/mbr/PartTypes.htm
var partitionTypes scalar.UToDescription = scalar.UToDescription{
0x00: "empty",
0x01: "12-bit FAT",
0x02: "XENIX root",
0x03: "XENIX /usr (obsolete)",
0x04: "16-bit FAT",
0x05: "Extended Partition",
0x06: "16-bit FAT, partition",
0x07: "Installable file systems: HPFS or NTFS. Also, QNX and Advanced Unix.",
0x08: "AIX bootable partition",
0x09: "AIX data partition",
0x0A: "Coherent swap partition, OPUS or OS/2 Boot Manager.",
0x0B: "32-bit FAT",
0x0C: "32-bit FAT, using INT 13 Extensions.",
0x0E: "16-bit FAT >= 32 MB, using INT 13 Extensions.",
0x0F: "Extended Partition, using INT 13 Extensions",
0x10: "OPUS",
0x11: "Hidden 12-bit FAT.",
0x12: "Compaq diagnostics.",
0x14: "Hidden 16-bit FAT",
0x16: "Hidden 16-bit FAT, partition >= 32 MB",
0x17: "Hidden IFS (HPFS, NTFS).",
0x18: "AST Windows swap file",
0x19: "Willowtech Photon coS",
0x1B: "Hidden 32-bit FAT",
0x1C: "Hidden 32-bit FAT, Ext INT 13",
0x1E: "Hidden 16-bit FAT >32 MB, Ext. INT 13 (PowerQuest specific)",
0x20: "Willowsoft Overture File System (OFS1)",
0x21: "reserved (HP Volume Expansion, SpeedStor variant)",
0x22: "Oxygen Extended",
0x23: "reserved (HP Volume Expansion, SpeedStor variant?)",
0x24: "NEC MS-DOS 3.x",
0x26: "reserved (HP Volume Expansion, SpeedStor variant?)",
0x31: "reserved (HP Volume Expansion, SpeedStor variant?)",
0x33: "reserved (HP Volume Expansion, SpeedStor variant?)",
0x34: "reserved (HP Volume Expansion, SpeedStor variant?)",
0x36: "reserved (HP Volume Expansion, SpeedStor variant?)",
0x38: "Theos",
0x3C: "PowerQuest Files Partition Format",
0x3D: "Hidden NetWare",
0x40: "VENIX 80286",
0x41: "Personal RISC Boot, PowerPC boot partition",
0x42: "Secure File System, Windows 2000/XP (NT 5)",
0x43: "Alternative Linux native file system (EXT2fs) PTS-DOS 6.70 & BootWizard: DR-DOS",
0x45: "Priam, EUMEL/Elan. ",
0x46: "EUMEL/Elan",
0x47: "EUMEL/Elan",
0x48: "EUMEL/Elan",
0x4A: "ALFS/THIN lightweight filesystem for DOS",
0x4D: "QNX",
0x4E: "QNX",
0x4F: "QNX, Oberon boot/data partition.",
0x50: "Ontrack Disk Manager, read-only partition, FAT partition (Logical sector size varies)",
0x51: "Ontrack Disk Manager, read/write partition, FAT partition (Logical sector size varies) Novell?",
0x52: "CP/M, Microport System V/386.",
0x53: "Ontrack Disk Manager, write-only",
0x54: "Ontrack Disk Manager 6.0 (DDO)",
0x55: "EZ-Drive 3.05",
0x56: "Golden Bow VFeature",
0x5C: "Priam EDISK",
0x61: "Storage Dimensions SpeedStor",
0x63: "GNU HURD, Mach, MtXinu BSD 4.2 on Mach, Unix Sys V/386, 386/ix.",
0x64: "Novell NetWare 286, SpeedStore.",
0x65: "Novell NetWare (3.11 and 4.1)",
0x66: "Novell NetWare 386",
0x67: "Novell NetWare",
0x68: "Novell NetWare",
0x69: "Novell NetWare 5+",
0x70: "DiskSecure Multi-Boot",
0x75: "IBM PC/IX",
0x80: "Minix v1.1 - 1.4a, Old MINIX (Linux).",
0x81: "Linux/Minix v1.4b+, Mitac Advanced Disk Manager.",
0x82: "Linux Swap partition, Prime or Solaris (Unix).",
0x83: "Linux native file systems (ext2/3/4, JFS, Reiser, xiafs, and others).",
0x84: "OS/2 hiding type 04h partition",
0x86: "NT Stripe Set, Volume Set?",
0x87: "NT Stripe Set, Volume Set?, HPFS FT mirrored partition.",
0x93: "Amoeba file system, Hidden Linux EXT2 partition (PowerQuest).",
0x94: "Amoeba bad block table",
0x99: "Mylex EISA SCSI",
0x9F: "BSDI",
0xA0: "Phoenix NoteBios Power Management 'Save to Disk', IBM hibernation.",
0xA1: "HP Volume Expansion (SpeedStor variant)",
0xA3: "HP Volume Expansion (SpeedStor variant)",
0xA4: "HP Volume Expansion (SpeedStor variant)",
0xA5: "FreeBSD/386",
0xA6: "OpenBSD",
0xA7: "NextStep Partition",
0xA9: "NetBSD",
0xAA: "Olivetti DOS with FAT12",
0xB0: "Bootmanager BootStar by Star-Tools GmbH",
0xB1: "HP Volume Expansion (SpeedStor variant)",
0xB3: "HP Volume Expansion (SpeedStor variant)",
0xB4: "HP Volume Expansion (SpeedStor variant)",
0xB6: "HP Volume Expansion (SpeedStor variant)",
0xB7: "BSDI file system or secondarily swap",
0xB8: "BSDI swap partition or secondarily file system",
0xBB: "PTS BootWizard (hidden) 4.0",
0xBC: "May be an Acronis 'Backup' or 'Secure Zone' partition, when labeled 'ACRONIS SZ' (FAT32, LBA mapped, primary).",
0xBE: "Solaris boot partition",
0xC0: "Novell DOS/OpenDOS/DR-OpenDOS/DR-DOS secured partition, or CTOS (reported by a client).",
0xC1: "DR-DOS 6.0 LOGIN.EXE-secured 12-bit FAT partition",
0xC2: "Reserved for DR-DOS 7+",
0xC3: "Reserved for DR-DOS 7+",
0xC4: "DR-DOS 6.0 LOGIN.EXE-secured 16-bit FAT partition",
0xC6: "DR-DOS 6.0 LOGIN.EXE-secured Huge partition, or: Corrupted FAT16 volume/stripe (V/S) set (Windows NT).",
0xC7: "Syrinx, Cyrnix, HPFS FT disabled mirrored partition, or: Corrupted NTFS volume/stripe set.",
0xC8: "Reserved for DR-DOS 7+",
0xC9: "Reserved for DR-DOS 7+",
0xCA: "Reserved for DR-DOS 7+",
0xCB: "Reserved for DR-DOS secured FAT32",
0xCC: "Reserved for DR-DOS secured FAT32X (LBA)",
0xCD: "Reserved for DR-DOS 7+",
0xCE: "Reserved for DR-DOS secured FAT16X (LBA)",
0xCF: "Reserved for DR-DOS secured Extended partition (LBA)",
0xD0: "Multiuser DOS secured (FAT12???)",
0xD1: "Old Multiuser DOS secured FAT12",
0xD4: "Old Multiuser DOS secured FAT16 (<= 32M)",
0xD5: "Old Multiuser DOS secured extended partition",
0xD6: "Old Multiuser DOS secured FAT16 (BIGDOS > 32 Mb)",
0xD8: "CP/M 86",
0xDB: "CP/M, Concurrent CP/M, Concurrent DOS, or CTOS (Convergent Technologies OS).",
0xDE: "Dell partition. Normally it contains a FAT16 file system of about 32 MB.",
0xDF: "BootIt EMBRM",
0xE1: "SpeedStor 12-bit FAT Extended partition, DOS access (Linux).",
0xE2: "DOS read-only (Florian Painke's XFDISK 1.0.4)",
0xE3: "SpeedStor (Norton, Linux says DOS R/O)",
0xE4: "SpeedStor 16-bit FAT Extended partition",
0xE5: "Tandy DOS with logical sectored FAT",
0xE6: "Storage Dimensions SpeedStor",
0xEB: "BeOS file system",
0xED: "Reserved for Matthias Paul's Spryt*x",
0xEE: "GPT Protective MBR followed by a GPT/EFI Header. Used to define a fake partition covering the entire disk.",
0xEF: "EFI/UEFI System Partition (or ESP)",
0xF1: "SpeedStor Dimensions (Norton,Landis)",
0xF2: "DOS 3.3+ second partition, Unisys DOS with logical sectored FAT.",
0xF3: "Storage Dimensions SpeedStor",
0xF4: "SpeedStor Storage Dimensions (Norton,Landis)",
0xF5: "Prologue",
0xF6: "Storage Dimensions SpeedStor",
0xFD: "Reserved for FreeDOS (http://www.freedos.org)",
0xFE: "LANstep, IBM PS/2 IML (Initial Microcode Load) partition, or...",
0xFF: "Xenix bad-block table",
}
Binary file added format/mbr/testdata/mbr.bin
Binary file not shown.
42 changes: 42 additions & 0 deletions format/mbr/testdata/mbr.fqtest
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
$ fq -d mbr dv mbr.bin
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: mbr.bin (mbr) 0x0-0x1ff.7 (512)
0x000|eb 63 90 90 90 90 90 90 90 90 90 90 90 90 90 90|.c..............| code_area: raw bits 0x0-0x1bd.7 (446)
* |until 0x1bd.7 (446) | |
| | | partition_table{}: 0x1be-0x1fd.7 (64)
| | | entry_1{}: 0x1be-0x1cd.7 (16)
0x1b0| 80 | . | boot_indicator: 128 (active) 0x1be-0x1be.7 (1)
0x1b0| 02| .| starting_chs_vals: "CHS(0, 2, 1)" 0x1bf-0x1c1.7 (3)
0x1c0|01 00 |.. |
0x1c0| cd | . | partition_type: 205 (Reserved for DR-DOS 7+) 0x1c2-0x1c2.7 (1)
0x1c0| 3f e0 ff | ?.. | ending_chs_vals: "CHS(ff, 3f, 20)" 0x1c3-0x1c5.7 (3)
0x1c0| 40 00 00 | @.. | starting_sector: "CHS(0, 40, 0)" 0x1c6-0x1c8.7 (3)
0x1c0| ec 26 79 00 | .&y. | partition_size: 7939820 0x1ca-0x1cd.7 (4)
| | | entry_2{}: 0x1ce-0x1dd.7 (16)
0x1c0| 00 | . | boot_indicator: 0 (inactive) 0x1ce-0x1ce.7 (1)
0x1c0| 3f| ?| starting_chs_vals: "CHS(ff, 3f, 20)" 0x1cf-0x1d1.7 (3)
0x1d0|e0 ff |.. |
0x1d0| ef | . | partition_type: 239 (EFI/UEFI System Partition (or ESP)) 0x1d2-0x1d2.7 (1)
0x1d0| 3f e0 ff | ?.. | ending_chs_vals: "CHS(ff, 3f, 20)" 0x1d3-0x1d5.7 (3)
0x1d0| 2c 27 79 | ,'y | starting_sector: "CHS(79, 2c, 27)" 0x1d6-0x1d8.7 (3)
0x1d0| 00 20 00 00 | . .. | partition_size: 8192 0x1da-0x1dd.7 (4)
| | | entry_3{}: 0x1de-0x1ed.7 (16)
0x1d0| 00 | . | boot_indicator: 0 (inactive) 0x1de-0x1de.7 (1)
0x1d0| 00| .| starting_chs_vals: "CHS(0, 0, 0)" 0x1df-0x1e1.7 (3)
0x1e0|00 00 |.. |
0x1e0| 00 | . | partition_type: 0 (empty) 0x1e2-0x1e2.7 (1)
0x1e0| 00 00 00 | ... | ending_chs_vals: "CHS(0, 0, 0)" 0x1e3-0x1e5.7 (3)
0x1e0| 00 00 00 | ... | starting_sector: "CHS(0, 0, 0)" 0x1e6-0x1e8.7 (3)
0x1e0| 00 00 00 00 | .... | partition_size: 0 0x1ea-0x1ed.7 (4)
| | | entry_4{}: 0x1ee-0x1fd.7 (16)
0x1e0| 00 | . | boot_indicator: 0 (inactive) 0x1ee-0x1ee.7 (1)
0x1e0| 00| .| starting_chs_vals: "CHS(0, 0, 0)" 0x1ef-0x1f1.7 (3)
0x1f0|00 00 |.. |
0x1f0| 00 | . | partition_type: 0 (empty) 0x1f2-0x1f2.7 (1)
0x1f0| 00 00 00 | ... | ending_chs_vals: "CHS(0, 0, 0)" 0x1f3-0x1f5.7 (3)
0x1f0| 00 00 00 | ... | starting_sector: "CHS(0, 0, 0)" 0x1f6-0x1f8.7 (3)
0x1f0| 00 00 00 00 | .... | partition_size: 0 0x1fa-0x1fd.7 (4)
0x1c0| 00 | . | unknown0: raw bits 0x1c9-0x1c9.7 (1)
0x1d0| 00 | . | unknown1: raw bits 0x1d9-0x1d9.7 (1)
0x1e0| 00 | . | unknown2: raw bits 0x1e9-0x1e9.7 (1)
0x1f0| 00 | . | unknown3: raw bits 0x1f9-0x1f9.7 (1)
0x1f0| 55 aa| U.| boot_record_sig: 0xaa55 0x1fe-0x1ff.7 (2)
1 change: 1 addition & 0 deletions pkg/interp/testdata/args.fqtest
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ jpeg Joint Photographic Experts Group file
json JSON
macho Mach-O macOS executable
matroska Matroska file
mbr Master Boot Record
mp3 MP3 file
mp3_frame MPEG audio layer 3 frame
mp4 ISOBMFF MPEG-4 part 12 and similar
Expand Down

0 comments on commit 467ede9

Please sign in to comment.