Skip to content

Commit

Permalink
Add deserialization support for appinfo.vdf V29
Browse files Browse the repository at this point in the history
appinfo.vdf V29 was introduced in Steam beta. This new version
introduces a space-saving optimization: instead of encoding each key
name in the binary VDF segment directly, an int64 identifier is instead
used for each key, with a table at the end of the 'appinfo.vdf' file
providing the mapping to actual key names.

Also see
SteamDatabase/SteamAppInfo@56b1fec

Fixes #462

Co-authored-by: Eamonn Rea <[email protected]>
  • Loading branch information
sonic2kk authored and Matoking committed Aug 3, 2024
1 parent 26166e0 commit 052a6ae
Showing 1 changed file with 45 additions and 6 deletions.
51 changes: 45 additions & 6 deletions steam/utils/appcache.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
>>> header, apps = parse_appinfo(open('/d/Steam/appcache/appinfo.vdf', 'rb'))
>>> header
{'magic': b"(DV\\x07", 'universe': 1}
{'magic': b")DV\\x07", 'universe': 1}
>>> next(apps)
{'appid': 5,
'size': 79,
Expand Down Expand Up @@ -43,6 +43,7 @@

uint32 = struct.Struct('<I')
uint64 = struct.Struct('<Q')
int64 = struct.Struct('<q')

def parse_appinfo(fp):
"""Parse appinfo.vdf from the Steam appcache folder
Expand All @@ -53,8 +54,9 @@ def parse_appinfo(fp):
:return: (header, apps iterator)
"""
# format:
# uint32 - MAGIC: "'DV\x07" or "(DV\x07"
# uint32 - MAGIC: "'DV\x07" or "(DV\x07" or b")DV\x07"
# uint32 - UNIVERSE: 1
# int64 - OFFSET TO KEY TABLE (added in ")DV\x07")
# ---- repeated app sections ----
# uint32 - AppID
# uint32 - size
Expand All @@ -63,17 +65,52 @@ def parse_appinfo(fp):
# uint64 - accessToken
# 20bytes - SHA1
# uint32 - changeNumber
# 20bytes - binary_vdf SHA1 (added in "(DV\x07"
# 20bytes - binary_vdf SHA1 (added in "(DV\x07")
# variable - binary_vdf
# ---- end of section ---------
# uint32 - EOF: 0
#
# ---- key table ----
# uint32 - Count of keys
# char[] - Null-terminated strings corresponding to field names

magic = fp.read(4)
if magic not in (b"'DV\x07", b"(DV\x07"):
if magic not in (b"'DV\x07", b"(DV\x07", b")DV\x07"):
raise SyntaxError("Invalid magic, got %s" % repr(magic))

universe = uint32.unpack(fp.read(4))[0]

key_table = None

appinfo_version = magic[0]
if appinfo_version >= 41: # b')'
# appinfo.vdf V29 and newer store list of keys in separate table at the
# end of the file to reduce size. Retrieve it and pass it to the VDF
# parser later.
key_table = []

key_table_offset = struct.unpack('q', fp.read(8))[0]
offset = fp.tell()
fp.seek(key_table_offset)
key_count = uint32.unpack(fp.read(4))[0]

# Read all null-terminated strings into a list
for _ in range(0, key_count):
field_name = bytearray()
while True:
field_name += fp.read(1)

if field_name[-1] == 0:
field_name = field_name[0:-1]
field_name = field_name.decode('utf-8', 'replace')

key_table.append(field_name)
break

# Rewind to the beginning of the file after the header:
# we can now parse the rest of the file.
fp.seek(offset)

def apps_iter():
while True:
appid = uint32.unpack(fp.read(4))[0]
Expand All @@ -91,10 +128,12 @@ def apps_iter():
'change_number': uint32.unpack(fp.read(4))[0],
}

if magic == b"(DV\x07":
if magic != b"'DV\x07":
app['data_sha1'] = fp.read(20)

app['data'] = binary_load(fp)
# 'key_table' will be None for older 'appinfo.vdf' files which
# use self-contained binary VDFs.
app['data'] = binary_load(fp, key_table=key_table)

yield app

Expand Down

0 comments on commit 052a6ae

Please sign in to comment.