Initial working catalog generation and reading scripts

Signed-off-by: Santosh Sivaraj <santosiv@in.ibm.com>

8 files changed, 386 insertions, 0 deletions

diff --git a/catalog.py b/catalog.py
new file mode 100644
index 0000000..be20a24
--- /dev/null
+++ b/catalog.py
@@ -0,0 +1,56 @@
+import struct
+import time
+import sys
+from datetime import datetime
+from common import *
+from events import pack_events
+from groups import pack_groups
+from formulae import pack_formulae
+
+def dump_schema(cat_file):
+    f = open(cat_file)
+    f.seek(0x1000)
+    s = f.read(PAGE_SIZE)
+    f.close()
+
+    return s
+
+def create_catalog(version, old_lid):
+    events, enum = pack_events('events.csv')
+    groups, gnum  = pack_groups('groups.csv')
+    formulae, fnum = pack_formulae('formulae.csv')
+    # we will not create the schema now, will extract the schema from the
+    # existing catalog file.
+    schema = dump_schema(old_lid)
+
+    schema_offset = 1
+    events_offset = schema_offset + len(schema) / PAGE_SIZE
+    events_length = len(events) / PAGE_SIZE
+    groups_offset = events_offset + len(events) / PAGE_SIZE
+    groups_length = len(groups) / PAGE_SIZE
+    formulae_offset = groups_offset + len(groups) / PAGE_SIZE
+    formulae_length = len(formulae) / PAGE_SIZE
+    # WARNING: schema len and offset is hardcoded here
+    header = struct.pack(">IIQ16s32xHHHxxHHHxxHHHxxHHHxx", 0x32347837, 64,
+                         version,
+                         datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S'),
+                         1, 1, 4, events_offset, events_length, enum,
+                         groups_offset, groups_length, gnum, formulae_offset,
+                         formulae_length, fnum)
+
+    return pad_page(header, PAGE_SIZE) + schema + events + groups + formulae
+
+if __name__ == "__main__":
+    if len(sys.argv) <= 3:
+        print "Usage: ./%s version old_lid out_file" % (sys.argv[0])
+        exit(1)
+
+    # get the version for the new build, and older catalog file to extract the
+    # schema data
+    c = create_catalog(int(sys.argv[1]), sys.argv[2])
+    padlen = 64 - len(c) / PAGE_SIZE
+    c += struct.pack('%dx' % (padlen * PAGE_SIZE))
+
+    f = open(sys.argv[3], 'wt')
+    f.write(c)
+    f.close()
diff --git a/common.py b/common.py
new file mode 100644
index 0000000..ec828b3
--- /dev/null
+++ b/common.py
@@ -0,0 +1,61 @@
+import struct
+
+PAGE_SIZE=4096
+
+def quotechars(chars):
+    return ''.join( ['.', c][c.isalnum()] for c in chars)
+
+def hexdump(chars, sep, width):
+    while chars:
+        line = chars[:width]
+        chars = chars[width:]
+        line = line.ljust( width, '\000' )
+        print "%s%s%s" % (sep.join( "%02x" % ord(c) for c in line ),
+                          sep, quotechars(line))
+
+def pack_text(text):
+    # Won't be fun if all goes well.. so here we might get empty texts, if so
+    # we just need to update the length field to the length of the length
+    # field. See catalog proposal document if you don't get this.
+
+    if len(text) == 0:
+        text_bin = struct.pack(">H", 2)
+    else:
+        l = len(text) + 1 + 2 # a null byte included
+
+        if l % 2 != 0:
+            l += 2 - l % 2
+
+        text_bin = struct.pack(">H%ds" % (l - 2), l, text)
+
+    return text_bin
+
+def pad16(dump):
+    # default is 16 bit length, the old formula structure had a bug, mentioning
+    # it as 32 bit structure length
+    tlen = len(dump) + 2
+    plen = 0
+    if tlen % 16 != 0:
+        plen = 16 - tlen % 16;
+        tlen += plen
+
+    slen = struct.pack(">H", tlen)
+
+    padding = struct.pack("%dx" % (plen))
+
+    return slen + dump + padding
+
+def pad_page(dump, page_size):
+    if len(dump) % page_size != 0:
+        # pad the remaining page
+        dump += struct.pack("%dx" % (PAGE_SIZE - len(dump) % PAGE_SIZE))
+
+        return dump
+
+def read_string(dump):
+    nlen = struct.unpack_from(">H", dump)[0]
+
+    print nlen
+    name = struct.unpack_from(">%ds" % (nlen - 2), dump[2:])
+
+    return name[0], nlen
diff --git a/events.py b/events.py
new file mode 100644
index 0000000..e1c6816
--- /dev/null
+++ b/events.py
@@ -0,0 +1,50 @@
+# This file packages all events into a binary file, which will be a part of the
+# catalog file.
+from struct import *
+import csv
+from common import *
+
+def event_pack(event):
+    header = pack(">xxBxHHHIHH", event['domain'], event['record_byte_offset'],
+                  event['record_length'], event['counter_offset'],
+                  event['flag'], event['primary_group_index'],
+                  event['group_count'])
+
+    e = header + pack_text(event['name']) + pack_text(event['description'])
+    e += pack_text(event['detailed_description'])
+
+    return pad16(e)
+
+def pack_events(events_csv):
+    events = ""
+
+    with open(events_csv) as csvfile:
+        reader = csv.DictReader(csvfile)
+        event = {}
+        count = 0
+        for row in reader:
+            count += 1
+            event['domain'] = int(row['domain'])
+            event['record_byte_offset'] = int(row['record byte offset'])
+            event['record_length'] = int(row['record length'])
+            event['counter_offset'] = int(row['counter offset'])
+            event['flag'] = int(row['flag'])
+            event['primary_group_index'] = int(row['primary group index'])
+            event['group_count'] = int(row['group count'])
+            event['name'] = row['name']
+            event['description'] = row['description']
+            event['detailed_description'] = row['detailed description']
+
+            events += event_pack(event)
+
+        return pad_page(events, PAGE_SIZE), count
+
+if __name__ == "__main__":
+    events = pack_events('events.csv')
+    if len(events) == 0:
+        print "Error in generating events binary dump"
+
+    f = open('events.bin', 'w')
+    f.write(events)
+    f.close()
+    hexdump(events, " ", 16)
diff --git a/formulae.py b/formulae.py
new file mode 100644
index 0000000..e64fd9c
--- /dev/null
+++ b/formulae.py
@@ -0,0 +1,50 @@
+# Pack the formulae into a binary dump to be used in the catalog
+from struct import *
+import csv
+from common import *
+
+
+def formula_pack(formula):
+    # First pack the header
+    header = pack(">xxIH6x", formula['flag'], formula['group'])
+
+    st = header + pack_text(formula['name']) + pack_text(formula['desc']) + pack_text(formula['formula'])
+
+    t = pad16(st)               # this also adds the length
+
+    return t
+
+
+def pack_formulae(formulae_csv):
+    formulae = ""
+
+    with open(formulae_csv) as csvfile:
+        reader = csv.DictReader(csvfile)
+        formula = {}
+        count = 0
+        for row in reader:
+            count += 1
+            formula['name'] = row['Formula Name']
+            formula['desc'] = row['Formula Description']
+            formula['formula'] = row['Formula']
+            if row['Grouped'] == 'y':
+                formula['flag'] = 0x04
+                formula['group'] = int(row['Group'])
+            else:
+                formula['flag'] = 0
+                formula['group'] = 0
+
+            formulae += formula_pack(formula)
+
+        return pad_page(formulae, PAGE_SIZE), count
+
+
+if __name__ == "__main__":
+    formulae = pack_formulae('formulae.csv')
+    if len(formulae) == 0:
+        print "Error in generating formulae binary dump"
+
+    f = open('formulae.bin', 'w')
+    f.write(formulae)
+    f.close()
+    hexdump(formulae, " ", 16)
diff --git a/groups.py b/groups.py
new file mode 100644
index 0000000..7818733
--- /dev/null
+++ b/groups.py
@@ -0,0 +1,46 @@
+from struct import *
+import csv
+from common import *
+
+def group_pack(group):
+    header = pack(">xxIBxHHBB", group['flag'], group['domain'], \
+                  group['event group offset'], group['event group length'], \
+                  group['schema index'], group['event count'])
+
+    indexes = pack(">16H", *eval(group['event indexes']))
+
+    g = header + indexes + pack_text(group['name']) + pack_text(group['description'])
+    return pad16(g)
+
+def pack_groups(groups_csv):
+    groups = ""
+
+    with open(groups_csv) as csvfile:
+        reader = csv.DictReader(csvfile)
+        group = {}
+        count = 0
+        for row in reader:
+            count += 1
+            group['flag'] = int(row['flag'])
+            group['domain'] = int(row['domain'])
+            group['event group offset'] = int(row['event group offset'])
+            group['event group length'] = int(row['event group length'])
+            group['schema index'] = int(row['schema index'])
+            group['event count'] = int(row['event count'])
+            group['event indexes'] = row['event indexes']
+            group['name'] = row['name']
+            group['description'] = row['description']
+
+            groups += group_pack(group)
+
+        return pad_page(groups, PAGE_SIZE), count
+
+if __name__ == "__main__":
+    groups = pack_groups('groups.csv')
+    if len(groups) == 0:
+        print "Error in generating groups binary dump"
+
+    f = open('groups.bin', 'w')
+    f.write(groups)
+    f.close()
+    hexdump(groups, " ", 16)
diff --git a/read_events.py b/read_events.py
new file mode 100644
index 0000000..f8ba9dd
--- /dev/null
+++ b/read_events.py
@@ -0,0 +1,41 @@
+from struct import *
+import csv
+from common import *
+
+NAMELEN_OFFSET = 0x14
+
+def read_event(event_dump):
+    event = {}
+    elen, event['domain'], event['record byte offset'], event['record length'], \
+        event['counter offset'], event['flag'], event['primary group index'], \
+        event['group count'] = unpack_from(">HxxBxHHHIHH", event_dump)
+
+    event['name'], nlen = read_string(event_dump[NAMELEN_OFFSET:])
+    event['description'], dlen = read_string(event_dump[(NAMELEN_OFFSET + nlen):])
+    event['detailed description'], ddlen = read_string(event_dump[(NAMELEN_OFFSET + nlen + dlen):])
+
+    return elen, event
+
+def read_events():
+    f = open('81e00610.v7.lid', 'r')
+    f.seek(0x2000, 0)
+    event_length = 0x31
+    dump = f.read(PAGE_SIZE * event_length)
+
+    i = 0
+    event_count = 0x532
+    offset = 0
+    f = open('events.csv', 'wt')
+    try:
+        writer = csv.writer(f)
+
+        for i in range(0, event_count):
+            elen, event = read_event(dump[offset:])
+            offset += elen
+            if i == 0:
+                writer.writerow((event.keys()))
+            writer.writerow((event.values()))
+    finally:
+        f.close()
+
+read_events()
diff --git a/read_formulae.py b/read_formulae.py
new file mode 100644
index 0000000..f9b1801
--- /dev/null
+++ b/read_formulae.py
@@ -0,0 +1,42 @@
+from struct import *
+import csv
+from common import *
+
+NAMELEN_OFFSET = 0x10
+
+def read_formula(formula_dump):
+    formula = {}
+    tlen, formula['flag'], formula['group'] = unpack_from(">HxxIH6x",
+                                                          formula_dump)
+
+    formula['name'], nlen = read_string(formula_dump[NAMELEN_OFFSET:])
+    formula['description'], dlen = read_string(formula_dump[(NAMELEN_OFFSET + nlen):])
+    formula['formula'], flen = read_string(formula_dump[(NAMELEN_OFFSET + nlen + dlen):])
+
+    print formula['name']
+
+    return tlen, formula
+
+def read_formulae():
+    f = open('81e00610.v7.lid', 'r')
+    f.seek(0x2d000, 0)
+    formula_length = 0x2
+    dump = f.read(PAGE_SIZE * formula_length)
+
+    i = 0
+    formula_count = 0x24
+    offset = 0
+    f = open('formula.csv', 'wt')
+    try:
+        writer = csv.writer(f)
+
+        for i in range(0, formula_count):
+            flen, formula = read_formula(dump[offset:])
+            offset += flen
+            if i == 0:
+                writer.writerow((formula.keys()))
+            writer.writerow((formula.values()))
+    finally:
+        f.close()
+
+read_formulae()
diff --git a/read_groups.py b/read_groups.py
new file mode 100644
index 0000000..5e8fd92
--- /dev/null
+++ b/read_groups.py
@@ -0,0 +1,40 @@
+from struct import *
+import csv
+from common import *
+
+NAMELEN_OFFSET = 0x30
+
+def read_group(group_dump):
+    group = {}
+    glen, group['flag'], group['domain'], group['event group offset'], \
+        group['event group length'], group['schema index'], \
+        group['event count'] = unpack_from(">HxxIBxHHBB", group_dump)
+    group['event indexes'] = unpack_from(">16H", group_dump[0x10:])
+    group['name'], nlen = read_string(group_dump[NAMELEN_OFFSET:])
+    group['description'], dlen = read_string(group_dump[(NAMELEN_OFFSET + nlen):])
+
+    return glen, group
+
+def read_groups():
+    f = open('81e00610.v7.lid', 'r')
+    f.seek(0x2a000, 0)
+    group_length = 5
+    dump = f.read(PAGE_SIZE * group_length)
+
+    i = 0
+    group_count = 0x8b
+    offset = 0
+    f = open('groups.csv', 'wt')
+    try:
+        writer = csv.writer(f)
+
+        for i in range(0, group_count):
+            glen, group = read_group(dump[offset:])
+            offset += glen
+            if i == 0:
+                writer.writerow((group.keys()))
+            writer.writerow((group.values()))
+    finally:
+        f.close()
+
+read_groups()