#!/usr/bin/env python3
import os
import sys
import argparse
import csv
import zlib
import base64
import collections
import statistics
import datetime
import subprocess
import pickle
import numpy as np
import matplotlib.pyplot as plt
import graphviz

wcw_thrs = [30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 95.0, 98.0, 99.0]

def hash_rgb_from_str(s):
    h = zlib.crc32( bytes(s, 'utf-8') )
    r = (h & 0x00FF00) >> 8
    g = (h & 0xFF0000) >> 16
    b = (h & 0x0000FF) >> 0
    return (float(r)/0xFF, float(g)/0xFF, float(b)/0xFF)

def hex_color(rgb):
    r = str(int(rgb[0] * 0xFF))
    g = str(int(rgb[1] * 0xFF))
    b = str(int(rgb[2] * 0xFF))
    return "#" + r + g + b

def median(l):
    if len(l) == 0:
        return 0.0
    return statistics.median(l)

def average(l):
    if len(l) == 0:
        return 0.0
    return statistics.mean(l)

def aad(l, avg):
    # aad (average absolute deviation)
    # https://en.wikipedia.org/wiki/Average_absolute_deviation
    if len(l) == 0:
        return 0.0
    sum = 0.0
    for v in l:
        sum = sum + abs(avg - v)
    return sum / len(l)

class ev_type:
    SCHED = 0
    IDLE  = 1
    MIG   = 2
    AWAKE = 3

class ev_time:
    time   = 0.0 
    cpu_id = 0

    def __init__(self, t, c):
        self.time = t
        self.cpu_id = c

    def __repr__(self):
        return "\t".join(["%.6f" % self.time, "[%d]" % self.cpu_id])

class task_waker_waiter:
    # class dict
    wcwn_dict = {}      # (waker, callstack, callstack typem waiter)   => count
    wtwn_dict = {}      # (waiter, callstak type, waker)               => count

    w_cwn_ddict = {}    # waker => (callstack, callstack type, waiter) => count
    w_twn_ddict = {}    # waker => (cs type, waiter)     => count

    wc_wn_ddict = {}    # waiter => (callstack, callstack type, waker) => count
    wt_wn_ddict = {}    # waiter => (cs type, waker)     => count
    
    waiter_dict = {}    # waiter                         => count
    waker_dict = {}     # waker                          => count

    cs_dict = {}        # cs, cst                        => count
    cst_dict = {}       # cst                            => count

    sep = "$@^__^@$"

    @classmethod
    def add_wcw(cls, t_waker, waiter_cs, t_waiter):
        # (waker, callstack, callstack type, waiter)     => count
        key = cls.sep.join( [str(t_waker), waiter_cs.cs, waiter_cs.cs_type, str(t_waiter)] )
        cls.wcwn_dict[key] = cls.wcwn_dict.get(key, 0) + 1
        # (waiter, callstak type, waker) => count
        key = cls.sep.join( [str(t_waker), waiter_cs.cs_type, str(t_waiter)] )
        cls.wtwn_dict[key] = cls.wtwn_dict.get(key, 0) + 1
                                                 
        # waker => (callstack, callstack type, waiter)   => count
        key = str(t_waker)
        dic = cls.w_cwn_ddict.get(key, None)
        if dic == None:
            dic = cls.w_cwn_ddict[key] = {}
        key = cls.sep.join( [waiter_cs.cs, waiter_cs.cs_type, str(t_waiter)] )
        dic[key] = dic.get(key, 0) + 1
        # waker => (cs type, waiter)     => count
        key = str(t_waker)
        dic = cls.w_twn_ddict.get(key, None) 
        if dic == None:
            dic = cls.w_twn_ddict[key] = {}
        key = cls.sep.join( [waiter_cs.cs_type, str(t_waiter)] )
        dic[key] = dic.get(key, 0) + 1

        # waiter => (callstack, callstack type, waker)    => count
        key = str(t_waiter)
        dic = cls.wc_wn_ddict.get(key, None)
        if dic == None:
           dic = cls.wc_wn_ddict[key] = {}
        key = cls.sep.join( [waiter_cs.cs, waiter_cs.cs_type, str(t_waker)] )
        dic[key] = dic.get(key, 0) + 1
        # waiter => (cs type, waker)     => count
        key = str(t_waiter)
        dic = cls.wt_wn_ddict.get(key, None)
        if dic == None:
            dic = cls.wt_wn_ddict[key] = {}
        key = cls.sep.join( [waiter_cs.cs_type, str(t_waker)] )
        dic[key] = dic.get(key, 0) + 1
                                                 
        # waiter                         => count
        key = str(t_waiter)
        cls.waiter_dict[key] = cls.waiter_dict.get(key, 0) + 1
        # waker                          => count
        key = str(t_waker)
        cls.waker_dict[key] = cls.waker_dict.get(key, 0) + 1

        # cs, cst                        => count
        key = cls.sep.join( [waiter_cs.cs, waiter_cs.cs_type] )
        cls.cs_dict[key] = cls.cs_dict.get(key, 0) + 1
        # cst                            => count
        key = waiter_cs.cs_type
        cls.cst_dict[key] = cls.cst_dict.get(key, 0) + 1

    @classmethod
    def get_wcwn(cls):
        data_s = list(cls.wcwn_dict.items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            toks = data.split(cls.sep)
            yield( [*toks, cnt] )

    @classmethod
    def get_wtwn(cls):
        data_s = list(cls.wtwn_dict.items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            toks = data.split(cls.sep)
            yield( [*toks, cnt] )

    @classmethod
    def get_waiter(cls):
        data_s = list(cls.waiter_dict.items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            yield( [data, cnt] )

    @classmethod
    def get_waker(cls):
        data_s = list(cls.waker_dict.items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            yield( [data, cnt] )

    @classmethod
    def get_cs(cls):
        data_s = list(cls.cs_dict.items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            toks = data.split(cls.sep)
            yield( [*toks, cnt] )

    @classmethod
    def get_cst(cls):
        data_s = list(cls.cst_dict.items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            yield( [data, cnt] )

    @classmethod
    def get_w_cwn(cls, waker_s):
        data_s = list(cls.w_cwn_ddict[waker_s].items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            toks = data.split(cls.sep)
            yield( [waker_s, *toks, cnt] )

    @classmethod
    def get_w_twn(cls, waker_s):
        data_s = list(cls.w_twn_ddict[waker_s].items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            toks = data.split(cls.sep)
            yield( [waker_s, *toks, cnt] )

    @classmethod
    def get_wc_wn(cls, waiter_s):
        data_s = list(cls.wc_wn_ddict[waiter_s].items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            toks = data.split(cls.sep)
            yield( [waiter_s, *toks, cnt] )

    @classmethod
    def get_wt_wn(cls, waiter_s):
        data_s = list(cls.wt_wn_ddict[waiter_s].items())
        data_s.sort(reverse = True, key = lambda x: x[1])
        for data, cnt in data_s:
            toks = data.split(cls.sep)
            yield( [waiter_s, *toks, cnt] )

class task_stat:
    # task
    task = None

    # ev_sched or ev_idle stat
    sched_num = 0

    sched_waits = None
    sched_wait_med = 0.0
    sched_wait_avg = 0.0
    sched_wait_aad = 0.0
    sched_wait_sum = 0.0

    sched_delays = None
    sched_delay_med = 0.0
    sched_delay_avg = 0.0
    sched_delay_aad = 0.0
    sched_delay_sum = 0.0

    sched_runs = None
    sched_run_med = 0.0
    sched_run_avg = 0.0
    sched_run_aad = 0.0
    sched_run_sum = 0.0
    # TODO: sched_ideal_runs = []

    # ev_mig stat
    migrated_num = 0

    # ev_awkae stat
    wake_num = 0
    wait_num = 0

    def __init__(self, task):
        self.task = task
        self.sched_num = 0
        self.sched_waits = []
        self.sched_wait_med = 0.0
        self.sched_wait_avg = 0.0
        self.sched_wait_aad = 0.0
        self.sched_wait_sum = 0.0

        self.sched_delays = []
        self.sched_delay_med = 0.0
        self.sched_delay_avg = 0.0
        self.sched_delay_aad = 0.0
        self.sched_delay_sum = 0.0

        self.sched_runs = []
        self.sched_run_med = 0.0
        self.sched_run_avg = 0.0
        self.sched_run_aad = 0.0
        self.sched_run_sum = 0.0

        self.migrated_num = 0

        self.wake_num = 0
        self.wait_num = 0

    def analyze_stat(self):
        # ev_sched or ev_idle?
        if self.task.is_idle() == True:
            sched_events = self.task.ctask_ty_events.get(ev_type.IDLE, collections.OrderedDict()) 
        else:
            sched_events = self.task.ctask_ty_events.get(ev_type.SCHED, collections.OrderedDict())

        for time, ev in sched_events.items():
            self.sched_waits.append(ev.period.wait_time)
            self.sched_delays.append(ev.period.sch_delay)
            self.sched_runs.append(ev.period.run_time)

        # ev_sched/ev_idle stat
        self.sched_num = len(sched_events)

        self.sched_wait_med = median(self.sched_waits)
        self.sched_wait_avg = average(self.sched_waits)
        self.sched_wait_aad = aad(self.sched_waits, self.sched_wait_avg)
        self.sched_wait_sum = sum(self.sched_waits)

        self.sched_delay_med = median(self.sched_delays)
        self.sched_delay_avg = average(self.sched_delays)
        self.sched_delay_aad = aad(self.sched_delays, self.sched_delay_avg)
        self.sched_delay_sum = sum(self.sched_delays)

        self.sched_run_med = median(self.sched_runs)
        self.sched_run_avg = average(self.sched_runs)
        self.sched_run_aad = aad(self.sched_runs, self.sched_run_avg)
        self.sched_run_sum = sum(self.sched_runs)

        # ev_mig stat
        migrated_events = self.task.ctask_ty_events.get(ev_type.MIG, collections.OrderedDict()) 
        self.migrated_num = len(migrated_events)

  
class ev_task:
    # per-class variables
    task_dict = {} 
    idle_task_name = "<idle>[0/0]"

    # per-instanace variables
    task_name = ""
    task_id   = 0
    parent_id = 0
    color = (0.0, 0.0, 0.0)

    # task indexes to events
    ctask_ty_events  = None  # cur_task:ty   -> [events]
    ctask_cs_events  = None  # cur_task:cs   -> [events]
    ctask_cst_events = None  # cur_task:cst  -> [events]
    ntask_ty_events  = None  # next_task:ty  -> [events]

    # task stat
    task_stat = None

    def __init__(self, tname, tid, pid):
        self.task_name = tname
        self.task_id = tid
        self.parent_id = pid
        self.color = hash_rgb_from_str( str(self) )

        self.ctask_ty_events  = collections.OrderedDict()
        self.ctask_cs_events  = collections.OrderedDict()
        self.ctask_cst_events = collections.OrderedDict()
        self.ntask_ty_events  = collections.OrderedDict()

        self.task_stat = task_stat(self)

    def __repr__(self):
        return "%s[%d/%d]" % (self.task_name, self.task_id, self.parent_id)

    @classmethod
    def get_instance(cls, tname, tid, pid):
        key = "%s[%d/%d]" % (tname, tid, pid)
        inst = cls.task_dict.get(key)
        if inst == None: 
            inst = cls(tname, tid, pid)
            cls.task_dict[key] = inst
        return inst

    @classmethod
    def find(cls, task_name):
        return cls.task_dict.get(task_name, None)

    @classmethod
    def analyze_stat(cls):
        for k, t in cls.task_dict.items():
            t.task_stat.analyze_stat()

    def is_idle(self):
        if self.task_id == 0 and self.parent_id == 0 and self.task_name == "<idle>":
            return True
        return False

    def link_ctask(self, ev, cs = None):
        # - cur_task:ty -> [events]
        ty_events = self.ctask_ty_events.get(ev.ty)
        if ty_events == None:
            ty_events = self.ctask_ty_events[ev.ty] = collections.OrderedDict()
        ty_events[ev.time.time] = ev

        if cs == None:
            return
        # - cur_task:cs -> [events]
        cs_events = self.ctask_cs_events.get(cs)
        if cs_events == None:
            cs_events = self.ctask_cs_events[cs] = collections.OrderedDict()
        cs_events[ev.time.time] = ev
        # - cur_task:cst -> [events]
        cst = cs.cs_type
        cst_events = self.ctask_cst_events.get(cst)
        if cst_events == None:
            cst_events = self.ctask_cst_events[cst] = collections.OrderedDict()
        cst_events[ev.time.time] = ev
        # - cs -> {ctasks}
        cs.link_ctask(self)

    def link_ntask(self, ev, cs = None):
        # - next_task:ty -> [events]
        ty_events = self.ntask_ty_events.get(ev.ty)
        if ty_events == None:
            ty_events = self.ntask_ty_events[ev.ty] = collections.OrderedDict()
        self.ntask_ty_events[ev.time.time] = ev

class ev_period:
    wait_time = 0.0
    sch_delay = 0.0
    run_time  = 0.0

    def __init__(self, w, s, r):
        self.wait_time = w
        self.sch_delay = s
        self.run_time  = r

    def __repr__(self):
        return "\t".join(["%.3f" % self.wait_time, 
                          "%.3f" % self.sch_delay,
                          "%.3f" % self.run_time])

class callstack_stat:
    # callstack
    callstack = None

    # callstack stats
    num_ctask = 0
    num_sched_events = 0

    def __init__(self, cs):
        self.callstack = cs
        self.num_sched_events = 0

    def analyze_stat(self):
        self.num_ctask = len(self.callstack.cs_ctasks)
        self.num_sched_events = 0
        for ctask in self.callstack.cs_ctasks:
            self.num_sched_events = self.num_sched_events + len(ctask.ctask_cs_events)

class callstack_type:
    type_dict = None

    @classmethod
    def init(cls):
        if cls.type_dict != None:
            return
        # dictionary for the know callsites
        d = [
                # wait for events
                ("epoll_", "epoll"),
                ("sys_poll", "poll"),
                ("unix_poll", "poll"),
                ("sock_poll", "poll"),
                ("sys_select", "select"),
                ("select_estimate_accuracy", "select"),
                ("sys_pselect", "pselect"),
                ("do_pselect", "pselect"),
                # pipe IO
                ("pipe_read", "pipe_read"),
                ("pipe_write", "pipe_write"),
                # futex
                ("futex_wake", "futex_wake"),
                ("futex_wait", "futex_wait"),
                ("sys_futex", "futex"),
                ("futex_", "futex"),
                # lock
                ("up_read", "unlock"),
                ("down_read", "lock"),
                ("up_write", "unlock"),
                ("down_write", "lock"),
                ("_unlock", "unlock"),
                ("_lock", "lock"),
                ("rwsem_wake", "unlock"),
                # file IO
                ("vfs_read", "fs_read"),
                ("sys_read", "fs_read"),
                ("do_read", "fs_read"),
                ("do_iter_read", "fs_read"),
                ("vfs_write", "fs_write"),
                ("sys_write", "fs_write"),
                ("do_write", "fs_write"),
                ("generic_perform_write", "fs_write"),
                ("jbd2_", "fs_write"),
                ("sys_openat", "fs_open"),
                ("blk_mq", "blk_mq:bio"),
                ("sys_ioctl", "ioctl"),
                ("_ioctl", "ioctl"),
                ("io_schedule", "io_schedule"),
                ("page_fault", "page_fault"),
                ("do_fault", "page_fault"),
                ("handle_mm_fault", "page_fault"),
                # network IO
                ("sys_recvmsg", "recvmsg"),
                ("sys_recvfrom", "recvfrom"),
                ("sys_sendto", "sendto"),
                ("unix_stream_sendmsg", "sendmsg"),
                ("unix_stream_recvmsg", "recvmsg"),
                ("unix_stream_read_generic", "recvmsg"),
                ("ip_queue_xmit", "net:xmit"),
                ("dev_queue_xmit", "net:xmit"),
                # system call
                ("sys_clone", "clone"),
                ("ret_from_fork", "fork"),
                ("do_exit", "exit"),
                ("do_wait", "wait"),
                ("sys_sched_yield", "yield"),
                ("do_nanosleep", "sleep"),
                ("sys_clock_nanosleep", "sleep"),
                ("usleep", "sleep"),
                ("syscall_", "syscall"),
                ("syscall_", "syscall"),
                # memory
                ("do_mprotect", "mprotect"),
                ("do_mmap", "mmap"),
                ("kmem_cache_alloc", "alloc"),
                ("vmalloc", "alloc"),
                ("kmalloc", "alloc"),
                ("alloc_pages", "alloc"),
                # sched preemption
                ("asm_sysvec_apic_timer_interrupt", "sched_preempt"),
                ("asm_sysvec_reschedule_ipi", "sched_preempt"),
                ("schedule_preempt_disabled", "sched_preempt"),
                ("preempt_schedule", "sched_preempt"),
                # sched wait
                ("schedule_timeout", "sched_wait"),
                ("wait_for_completion", "sched_wait"),
                # others
                ("try_to_wake_up", "try_to_wake_up"),
                ("asm_common_interrupt", "hw_interrupt"),
                ]
        cls.type_dict = d 

    @classmethod
    def get_type(cls, cs):
        if cls.type_dict == None:
            cls.init()
        for k, t in cls.type_dict:
            if cs.find(k) != -1:
                return t
        return "[unknown]" # + cs


class ev_callstack:
    # per-class variables
    cs_dict = {} 
    cst_dict = {}

    # per-instanace variables
    cs  = ""
    cs_type = ""
    color = (0.0, 0.0, 0.0)

    # callstack to caller tasks
    cs_ctasks = None

    # callstack stat
    callstack_stat = None

    @classmethod
    def get_instance(cls, cs):
        inst = cls.cs_dict.get(cs)
        if inst == None: 
            # create an instance
            inst = cls(cs)
            # add it to the cs_dict
            cls.cs_dict[cs] = inst
            # add it to the cst_dict
            cst_set = cls.cst_dict.get(inst.cs_type)
            if cst_set == None: 
                cst_set = cls.cst_dict[inst.cs_type] = set()
            cst_set.add(inst)
        return inst

    @classmethod
    def find(cls, cs):
        return cls.cs_dict.get(cs, None)

    @classmethod
    def analyze_stat(cls):
        for k, cs in cls.cs_dict.items():
            cs.callstack_stat.analyze_stat()

    def link_ctask(self, ctask):
        self.cs_ctasks.add(ctask)

    def link_ntask(self, ntask):
        self.cs_ntasks.add(ntask)

    def __init__(self, cs):
        self.cs = cs
        self.cs_type = callstack_type.get_type(cs)
        self.color = hash_rgb_from_str( str(self) )
        self.cs_ctasks = set()
        self.callstack_stat = callstack_stat(self)

    def __repr__(self):
        return self.cs

class ev_sched:
    # per-class variables
    ty = ev_type.SCHED
    color =  hash_rgb_from_str( str(ev_type.SCHED) )

    # per-instanace variables
    time   = None
    period = None
    cur_task  = None
    next_task = None 
    callstack = None

    def __init__(self, time, period, cur_task, next_task, callstack):
        self.time = time
        self.period = period

        self.cur_task = cur_task
        self.callstack = callstack
        self.next_task = next_task

        self.cur_task.link_ctask(self, self.callstack)
        self.next_task.link_ntask(self, self.callstack)

    def __repr__(self):
        return "\t".join(["S", str(self.time), str(self.cur_task), str(self.period), 
                          "=>", str(self.next_task), str(self.callstack)])

class ev_idle:
    # per-class variables
    ty = ev_type.IDLE
    color =  hash_rgb_from_str( str(ev_type.IDLE) )

    # per-instanace variables
    time = None
    period = None
    cur_task  = None
    next_task = None 

    def __init__(self, time, period, cur_task, next_task):
        self.time = time
        self.period = period

        self.cur_task = cur_task
        self.next_task = next_task

        self.cur_task.link_ctask(self)
        self.next_task.link_ntask(self)

    def __repr__(self):
        return "\t".join(["I", str(self.time), str(self.cur_task), str(self.period), 
                          "=>", str(self.next_task)])

class ev_mig:
    # per-class variables
    ty = ev_type.MIG
    color =  hash_rgb_from_str( str(ev_type.MIG) )

    # per-instanace variables
    time   = None
    cur_task  = None
    next_task = None 
    to_cpu_id = 0

    def __init__(self, time, cur_task, migrated_task, to_cpu_id):
        self.time = time
        self.to_cpu_id = to_cpu_id

        self.cur_task = migrated_task # migrated task should be the key
        self.next_task = cur_task     # next task is indeed an initiator

        self.cur_task.link_ctask(self)
        self.next_task.link_ntask(self)

    def __repr__(self):
        return "\t".join(["M", str(self.time), str(self.cur_task), 
                          "=>", str(self.next_task)])

class ev_awake:
    # per-class variables
    ty = ev_type.AWAKE
    color =  hash_rgb_from_str( str(ev_type.AWAKE) )

    # per-instanace variables
    time   = None
    cur_task  = None
    next_task = None 

    def __init__(self, time, cur_task, awakened_task):
        self.time = time

        self.cur_task = awakened_task # awakened task (waiter) should be the key
        self.next_task = cur_task     # next task is indeed an initiator (waker)

        self.cur_task.link_ctask(self)
        self.next_task.link_ntask(self)

    def __repr__(self):
        return "\t".join(["A", str(self.time), str(self.cur_task), 
                          "=>", str(self.next_task)])

class pearson_corr:
    data_label = ""
    data_names = None

    x_label = None
    y_label = None
    x_data  = None
    y_data  = None
    pcc     = None      # non-weighted correlation coefficient

    w_label = None
    w_data = None
    wcc      = None     # weighted correlation coefficient

    def __init__(self, x, y, d, w = None):
        self.data_label = d
        self.data_names = None

        self.x_label = x
        self.y_label = y
        self.x_data  = None
        self.y_data  = None
        self.pcc     = None

        self.w_label = w
        self.w_data = None
        self.wcc = None

    def calc_pcc(self):
        # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient
        self.pcc = np.corrcoef(np.array(self.x_data), np.array(self.y_data))[0,1]

    def calc_wcc(self):
        # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient#Weighted_correlation_coefficient
        def w_m(x, w):
            return np.sum(x * w) / np.sum(w)

        def w_cov(x, y, w):
            return np.sum(w * (x - w_m(x, w)) * (y - w_m(y, w))) / np.sum(w)

        x = np.array(self.x_data)
        y = np.array(self.y_data)
        w = np.array(self.w_data)
        self.wcc = w_cov(x, y, w) / np.sqrt(w_cov(x, x, w) * w_cov(y, y, w))

class corr_set:
    name = ""
    question = ""
    corrs = None

    def __init__(self, name, question, corrs):
        self.name = name
        self.question = question
        self.corrs = corrs


class csv_data:
    names = None
    stats = None
    formats = None

    def __init__(self):
        self.names = []
        self.stats = []
        self.formats = []

class analysis_results:
    sched_events = None

    # system-wide task and idle stats
    sw_task_stat = None
    sw_task_csv = None

    sw_idle_stat = None
    sw_idle_csv = None

    # pearson correlations
    corr_sets = None

    # task status using per-task average
    task_csv = None

    # callstack 
    cs_csv = None

    # callstack types
    cst_csv = None

    # waker-cs-waiter
    wcw_csv = None

    # waker-cst-waiter
    wtw_csv = None

    # waker => cs-watier
    w_cwn_csv = None

    # waker => cst- watier
    w_twn_csv = None

    # waiter => cs-waker
    wc_wn_csv = None

    # waiter => cst- waker
    wt_wn_csv = None

    # waiter
    waiter_csv = None

    # waker stats
    waker_csv = None

    # cs stats
    wcw_cs_csv = None

    # cst stats
    wtw_cst_csv = None


    def __init__(self):
        # pearson correlations
        self.corr_sets = []

        # system-wide task and idle stat
        self.sw_task_stat = ev_task("system-wide task info", 0, 0).task_stat
        self.sw_task_csv = csv_data()

        self.sw_idle_stat = None
        self.sw_idle_csv = csv_data()

        # task status using per-task average
        self.task_csv = csv_data()
        # callstack 
        self.cs_csv = csv_data()
        # callstack types
        self.cst_csv = csv_data()
        # waker-cs-waiter
        self.wcw_csv = csv_data()
        # waker-cst-waiter
        self.wtw_csv = csv_data()
        # waker => cs-watier
        self.w_cwn_csv = csv_data()
        # waker => cst- watier
        self.w_twn_csv = csv_data()
        # waiter => cs-waker
        self.wc_wn_csv = csv_data()
        # waiter => cst- waker
        self.wt_wn_csv = csv_data()
        # waiter
        self.waiter_csv = csv_data()
        # waker stats
        self.waker_csv = csv_data()
        # cs stats
        self.wcw_cs_csv = csv_data()
        # cst stats
        self.wtw_cst_csv = csv_data()

    def aggregate_system_wide_task_stat(self):
        # task stat
        sn, mn, w1n, w2n = 0, 0, 0, 0
        w, d, r = [], [], []
        for t in ev_task.task_dict.values():
            sn = sn + t.task_stat.sched_num
            mn = mn + t.task_stat.migrated_num
            w1n = w1n + t.task_stat.wake_num
            w2n = w2n + t.task_stat.wait_num

            w = w + t.task_stat.sched_waits
            d = d + t.task_stat.sched_delays
            r = r + t.task_stat.sched_runs

        self.sw_task_stat.sched_num = sn
        self.sw_task_stat.migrated_num = mn
        self.sw_task_stat.wake_num = w1n
        self.sw_task_stat.wait_num = w2n

        self.sw_task_stat.sched_waits = w
        self.sw_task_stat.sched_wait_med = median(w)
        self.sw_task_stat.sched_wait_avg = average(w)
        self.sw_task_stat.sched_wait_aad = aad(w, self.sw_task_stat.sched_wait_avg)
        self.sw_task_stat.sched_wait_sum = sum(w)

        self.sw_task_stat.sched_delays = d
        self.sw_task_stat.sched_delay_avg = average(d)
        self.sw_task_stat.sched_delay_aad = aad(d, self.sw_task_stat.sched_delay_avg)
        self.sw_task_stat.sched_delay_sum = sum(d)

        self.sw_task_stat.sched_runs = r
        self.sw_task_stat.sched_run_avg = average(r)
        self.sw_task_stat.sched_run_aad = aad(r, self.sw_task_stat.sched_run_avg)
        self.sw_task_stat.sched_run_sum = sum(r)

        # idle stat
        self.sw_idle_stat = ev_task.find(ev_task.idle_task_name).task_stat


class pickled_jar:
    task_dict = None
    cs_dict = None
    sched_events = None

    def __init__(self, se):
        self.task_dict = ev_task.task_dict
        self.cs_dict = ev_callstack.cs_dict
        self.sched_events = se


def parse_ev_time(toks):
    time = float(toks[0])
    cpu_id = int(toks[1][1:-1])
    return ev_time(time, cpu_id), toks[2:]
    
def parse_ev_type(toks):
    if toks[0] == "s":
        return ev_type.SCHED, toks[1:]
    elif toks[0] == "i":
        return ev_type.IDLE, toks[1:]
    elif toks[0] == "m":
        return ev_type.MIG, toks[1:]
    else:
        return ev_type.AWAKE, toks

def parse_ev_task(toks):
    if toks[0] == "<idle>" or toks[0] == "swapper": 
        return ev_task.get_instance(toks[0], 0, 0), toks[1:]
    
    tstr = ""
    for i, t in enumerate(toks):
        if t[-1] == "]":
            tstr = tstr + t
            toks = toks[i+1:]
            break
        tstr = tstr + t + " "

    tsk_tok = tstr[:-1].split("[")
    tname = tsk_tok[0]
    tid, pid = 0, 0
    if len(tsk_tok) == 2:
        pid_tok = tsk_tok[1].split("/")
        tid = int(pid_tok[0])
        if len(pid_tok) == 2: 
            pid = int(pid_tok[1])
    return ev_task.get_instance(tname, tid, pid), toks

def parse_ev_period(toks):
    w = float(toks[0])
    s = float(toks[1])
    r = float(toks[2])
    return ev_period(w, s, r), toks[3:]

def parse_ev_callstack(toks):
    cs = ' '.join(toks)
    return ev_callstack.get_instance(cs), []

def parse_ev_sched(toks, time):
    cur_task, toks = parse_ev_task(toks)
    period, toks = parse_ev_period(toks)
    next_task, toks = parse_ev_task(toks[1:]) # toks[0] should be 'next:'
    callstack, toks = parse_ev_callstack(toks)
    return ev_sched(time, period, cur_task, next_task, callstack)

def parse_ev_idle(toks, time):
    cur_task, toks = parse_ev_task(toks)
    period, toks = parse_ev_period(toks) 
    next_task, toks = parse_ev_task(toks[1:]) # toks[0] should be 'next:'
    return ev_idle(time, period, cur_task, next_task)

def parse_ev_mig(toks, time):
    cur_task, toks = parse_ev_task(toks)
    migrated_task, toks = parse_ev_task(toks[1:])  # toks[0] should be "migrated:"
    to_cpu_id = int(toks[3]) # toks[] = ["cpu", from_cpu_id, "=>", to_cpu_id]
    return ev_mig(time, cur_task, migrated_task, to_cpu_id)

def parse_ev_awake(toks, time):
    cur_task, toks = parse_ev_task(toks)
    awakened_task, toks = parse_ev_task(toks[1:])  # toks[0] should be "awakened:"
    return ev_awake(time, cur_task, awakened_task)

def parse_ev_line(line):
    parsers = {ev_type.SCHED: parse_ev_sched, ev_type.IDLE: parse_ev_idle, 
               ev_type.MIG: parse_ev_mig,     ev_type.AWAKE: parse_ev_awake,}

    toks = line.split()
    if len(toks) == 0:
        return None

    time, toks = parse_ev_time(toks)
    ty, toks = parse_ev_type(toks)
    return parsers[ty](toks, time)

def skip_column_header(f):
    f.readline()
    f.readline()
    f.readline()

def get_schedmon_log_name(args, kind):
    log = os.path.join(args.logdir,
                       args.prefix + "-schedmon-" + kind+ "__.log")
    return log

def parse_sched_events(f):
    # - header
    skip_column_header(f)

    # - scheduling events
    events = collections.OrderedDict()
    for line in f:
        ev = parse_ev_line(line)
        if ev == None:
            break
        events[ev.time.time] = ev
    return events

def is_newer(file_new, file_old):
    if os.path.isfile(file_new) == False:
        return False
    return os.path.getmtime(file_new) > os.path.getmtime(file_old)

def load_pickle(args, log_fil):
    if args.pickle == False:
        return None

    pickle_fil = get_outfile_name(args, "pickled_jar__", "pickle")
    if is_newer(pickle_fil, log_fil):
        with open(pickle_fil, mode="rb") as pf:
            pj = pickle.load(pf)
            # init class variables
            ev_task.task_dict = pj.task_dict
            ev_callstack.cs_dict = pj.cs_dict
            return pj.sched_events
    return None

def save_pickle(args, sched_events):
    if args.pickle == False:
        return

    pickle_fil = get_outfile_name(args, "pickled_jar__", "pickle")
    pj = pickled_jar(sched_events)
    with open(pickle_fil, mode="wb") as f:
        pickle.dump(pj, f)

def parse_schedmon_timehist(args, results):
    # get a log file name
    log_fil = get_schedmon_log_name(args, "timehist_full")

    # first try to load the pickled events
    sched_events = load_pickle(args, log_fil)
    if sched_events != None:
        return  sched_events

    # if nothing pickled, parse the log and pickle it
    with open(log_fil, "r") as f:
        # try to load the pickled log first
        sched_events = load_pickle(args, log_fil)

        # parse and pickle the log
        if sched_events == None:
            sched_events = parse_sched_events(f)
            save_pickle(args, sched_events)
        # - TODO: Runtime summary
        # - TODO: Terminated tasks:
        # - TODO: Idle stats:
        # - TODO: Overall smmary

    results.sched_events = sched_events

def filter_sched_events(events, filter_fn, filter_data):
    filtered_events = collections.OrderedDict()

    for time, ev in events.items():
        if filter_fn(ev, filter_data) == True:
            filtered_events[time] = ev
    rebuild_indexes(filtered_events)

    return filtered_events

def is_meaningful_task(args, task_stat):
    # filter out statically meaningless tasks
    if task_stat.sched_num < args.minsched:
        return False
    # filter out never scheduled tasks
    if task_stat.sched_num == 0 and task_stat.migrated_num == 0:
        return False
    return True


def analyze_task_idle_stat(args, r):
    def convert_task_stat_to_csv_stat(s):
        t_info = [str(s.task), s.sched_num, s.migrated_num, s.wake_num, s.wait_num]
        w_stat = [s.sched_wait_med, s.sched_wait_avg, s.sched_wait_aad, s.sched_wait_sum]
        d_stat = [s.sched_delay_med, s.sched_delay_avg, s.sched_delay_aad, s.sched_delay_sum]
        r_stat = [s.sched_run_med, s.sched_run_avg, s.sched_run_aad, s.sched_run_sum]

        return t_info + w_stat + d_stat + r_stat

    # per-task stats 
    ev_task.analyze_stat()
    # system-wide task/idle stat
    r.aggregate_system_wide_task_stat()

    # collect task/idle stats using per-task average
    stat_names = ["task_name", "num_sched", "num_mig", "num_wake", "num_wait", # 0--4
                  "wait_time_med", "wait_time_avg",
                  "wait_time_aad", "wait_time_total", # 5--8
                  "sched_delay_med", "sched_delay_avg",
                  "sched_delay_aad", "sched_delay_total", # 9--12
                  "runtime_med", "runtime_avg",
                  "runtime_aad", "runtime_total", # 13--16
                  "rank_nsched", "rank_nmig", "rank_runtime", "score", # 17--20
                  ]
    task_stats = []
    idle_stats = []
    for k, t in ev_task.task_dict.items():
        s = t.task_stat

        # filter out statically meaningless tasks
        if is_meaningful_task(args, s) == False:
            continue

        stat = convert_task_stat_to_csv_stat(s)

        if t.is_idle():
            idle_stats.append(stat + [0, 0, 0, 0])
        else:
            task_stats.append(stat)

    # calc score
    # - by number of sched
    task_stats.sort(reverse = True, key = lambda x: x[1])
    for i, s in enumerate(task_stats):
        s.append(i) # 17
    # - by number of migration
    task_stats.sort(reverse = True, key = lambda x: x[2])
    for i, s in enumerate(task_stats):
        s.append(i) # 18
    # - by total runtime
    task_stats.sort(reverse = True, key = lambda x: x[16])
    for i, s in enumerate(task_stats):
        s.append(i) # 19
    # - score
    for s in task_stats:
        ns, nm, rt = s[17], s[18], s[19]
        score = ns*ns + (rt*rt)/2 + (nm*nm)/8
        s.append(score) #20

    # soft by score
    task_stats.sort(key = lambda x: x[20])
    r.task_csv.names = stat_names
    r.task_csv.formats = ["{0:^30}"] + ["{0:>14}"] * (len(r.task_csv.names) - 1)
    r.task_csv.stats = task_stats

    r.sw_idle_csv.names = stat_names
    r.sw_idle_csv.formats = ["{0:^30}"] + ["{0:>14}"] * (len(r.sw_idle_csv.names) - 1)
    r.sw_idle_csv.stats = idle_stats

    # system-wide task info
    r.sw_task_csv.names = stat_names
    r.sw_task_csv.formats = ["{0:^30}"] + ["{0:>14}"] * (len(r.sw_task_csv.names) - 1)
    r.sw_task_csv.stats = [ convert_task_stat_to_csv_stat(r.sw_task_stat) ]


def get_outfile_name(args, kind, suffix):
    ofname = os.path.join(args.logdir, args.prefix + 
                          "-schedinsight-" + kind + "." + suffix)
    return ofname

def get_outfile_name_x(args, kind, suffix):
    odir = args.logdir
    fname = args.prefix + "-schedinsight-" + kind 
    return odir, fname, suffix

def reset_plot():
    plt.clf()
    plt.style.use('default')
    plt.rcParams['font.size'] = 7

def plot_event_chart(args, plot_name, events):
    ty_map = {}
    ctask_map = {}
    ntask_map = {}
    cs_map = {}

    # build data for plot
    def append_color(m, c, i):
        if m.get(c) == None:
            m[c] = []
        m[c].append(i)

    for i, (time, ev) in enumerate(events.items()):
        # - type
        append_color(ty_map, ev.color, i)
        append_color(ctask_map, ev.cur_task.color, i)
        append_color(ntask_map, ev.next_task.color, i)
        if ev.ty == ev_type.SCHED: 
            append_color(cs_map, ev.callstack.color, i)
        # TODO: not very useful if there are more than 1000 events
        # TODO: extend the inerface to time range

    # plot events
    # - prepare canvas
    reset_plot()
    fig, axs = plt.subplots(nrows=4, ncols=1, figsize=(7, 3.5), tight_layout=True)

    # - plot type
    def plot_map(ax, m, title):
        ax.tick_params(left = False, right = False , labelleft = False , 
                       labelbottom = False, bottom = False)
        for c, x_data in m.items():
            ax.bar(x_data, [1] * len(x_data), align='edge', width=1, color = c)
        ax.set_ylim(bottom = 0, top = 1)
        ax.set_ylabel(title)

    plot_map(axs[0], ty_map, "event type")
    plot_map(axs[1], ctask_map, "current task")
    plot_map(axs[2], cs_map, "callstack")
    plot_map(axs[3], ntask_map, "next task")

    # - decoration
    axs[0].tick_params(labeltop = True, top = True)
    axs[3].tick_params(labelbottom = True, bottom = True)
    plt.subplots_adjust(hspace = 0)

    # - save to the file
    fig_name = get_outfile_name(args, plot_name, args.imgtype)
    plt.savefig(fig_name)
    plt.close()

def gen_stat_in_csv(csv, f):
    col_names = csv.names
    col_formats = csv.formats
    tuples = csv.stats
    def get_sep(c, ncol):
        if c == (ncol - 1):
            return "\n"
        else:
            return ", "

    def is_float(n):
        try: 
            return int(n) != float(n)
        except ValueError:
            return False

    # column header
    ncol = len(col_names)
    for i, (fmt, cname) in enumerate(zip(col_formats, col_names)):
        print(fmt.format(cname), end=get_sep(i, ncol), file = f)

    # stat tuples 
    for tup in tuples: 
        for i, (fmt, cdata) in enumerate(zip(col_formats, tup)):
            if is_float(cdata):
                print(fmt.format("%.8f" % cdata), end=get_sep(i, ncol), file = f)
            else:
                print(fmt.format(cdata), end=get_sep(i, ncol), file = f)

def plot_dist_fig(args, plot_name, sorted_values, title, x_label, y_label):
    # prepare canvas
    reset_plot()
    fig, axs = plt.subplots(nrows=3, ncols=1, figsize=(3, 8), tight_layout=True)

    # 1) violin plot
    # - plot
    ax = axs[0]
    violin = ax.violinplot([sorted_values], showmeans=True,  quantiles=[0.1, 0.5, 0.9])
    violin['bodies'][0].set_facecolor( hash_rgb_from_str(y_label) )
    violin['cmeans'].set_edgecolor('red')
    # - set title
    ax.set_title(title)
    # - decoration
    ax.set_xlabel(x_label)
    ax.set_ylabel(y_label)

    # 2) distribution line plot
    # - plot
    ax = axs[1]
    ax.plot(range(len(sorted_values)), sorted_values, linewidth=1, color='black')
    # - decoration
    ax.set_xlabel(x_label)
    ax.set_ylabel(y_label)

    # 3) histogram plot
    # - plot
    ax = axs[2]
    ax.hist(sorted_values, bins = 30)
    # - decoration
    max_hist = len(sorted_values)/3
    ax.set_ylim(bottom = 0, top = max_hist)
    ax.set_xlabel(x_label)
    ax.set_ylabel(y_label)

    # - save to the file
    fig_name = get_outfile_name(args, plot_name, args.imgtype)
    plt.savefig(fig_name)
    plt.close()

def plot_corr_figs(args, corrs, plot_name):
    # prepare canvas
    reset_plot()
    fig, axs = plt.subplots(nrows=1, ncols=len(corrs), 
                            figsize=(3*len(corrs), 3), tight_layout=True)

    # scatter plot
    for i, corr in enumerate(corrs):
        if len(corrs) > 1:
            ax = axs[i]
        else:
            ax = axs
        colors = []
        for name in corr.data_names:
            colors.append( hash_rgb_from_str(name) )
        ax.scatter(corr.x_data, corr.y_data, c=colors, marker="x")
        ax.set(xlabel=corr.x_label, ylabel=corr.y_label)
        ax.title.set_text("pcc=" + "{:.2f}".format(corr.pcc) + " " + \
                          "wcc=" + "{:.2f}".format(corr.wcc))

    # - save to the filewineserver
    fig_name = get_outfile_name(args, plot_name, args.imgtype)
    plt.savefig(fig_name)
    plt.close()

def transpose_2d_list(ll):
    return list(zip(*ll))

def plot_task_dist_figs(args, stat_names, task_stats):
    stat_cols = transpose_2d_list(task_stats)
    for name, stat in zip(stat_names[1:-4], stat_cols[1:-4]):
        plot_dist_fig(args, "task_dist_" + name, sorted(stat), name, "", "")

def plot_per_task_stat_over_time(args, ts, title, plot_name, x_size = 3, y_size = 14):
    # prepare canvas
    reset_plot()
    fig, axs = plt.subplots(nrows=12, ncols=1, figsize=(x_size, y_size), tight_layout=True)

    def plot_filled_line(ax, data, c, y_label):
        l = len(data)
        ax.fill_between(range(l), data, color = c)
        ax.set_ylabel(y_label)

    def plot_filled_violin(ax, data, c, y_label):
        l = len(data)
        violin = ax.violinplot([data], showmeans=True,  quantiles=[0.1, 0.5, 0.9])
        violin['bodies'][0].set_facecolor(c)
        violin['cmeans'].set_edgecolor('red')
        ax.set_ylabel(y_label)

    def plot_filled_histogram(ax, data, c, y_label):
        ax.hist(data, bins = 30, color=c, ec=c)
        max_hist = len(data)/3
        ax.set_ylim(bottom = 0, top = max_hist)
        ax.set_ylabel(y_label)

    # time serises of wait time, sched delay, and runtimes
    plot_filled_line(axs[0], ts.sched_waits, 
                     ts.task.color, "wait_time:ts")
    plot_filled_line(axs[1], ts.sched_delays, 
                     ts.task.color, "sched_delay:ts")
    plot_filled_line(axs[2], ts.sched_runs, 
                     ts.task.color, "runtime:ts")

    # sorted distribution of wait time, sched delay, and runtimes
    plot_filled_line(axs[3], sorted(ts.sched_waits.copy()), 
                     ts.task.color, "wait_time:dist")
    plot_filled_line(axs[4], sorted(ts.sched_delays.copy()), 
                     ts.task.color, "sched_delay:dist")
    plot_filled_line(axs[5], sorted(ts.sched_runs.copy()), 
                     ts.task.color, "runtime:dist")

    # decoration
    axs[0].set_title(title)

    # violin plot for wait time, sched delay, and runtimes
    plot_filled_violin(axs[6], ts.sched_waits, 
                       ts.task.color, "wait_time")
    plot_filled_violin(axs[7], ts.sched_delays, 
                       ts.task.color, "sched_delay")
    plot_filled_violin(axs[8], ts.sched_runs, 
                       ts.task.color, "runtime")

    # histogram for wait time, sched delay, and runtimes
    plot_filled_histogram(axs[9], ts.sched_waits, 
                          ts.task.color, "wait_time")
    plot_filled_histogram(axs[10], ts.sched_delays, 
                          ts.task.color, "sched_delay")
    plot_filled_histogram(axs[11], ts.sched_runs, 
                          ts.task.color, "runtime")

    plt.subplots_adjust(hspace = 0)

    # save to the file
    fig_name = get_outfile_name(args, plot_name, args.imgtype)
    plt.savefig(fig_name)
    plt.close()

def plot_pie_chart(args, name_list, cnt_list, plot_name):
    # prep data
    total_cnt = sum(cnt_list)
    label_list = list( map(lambda t: "%s (%.2f%s)" % 
                           (t[0], 100.0 * t[1] / total_cnt, "%"), 
                           zip(name_list, cnt_list)) )
    rgb_list = list( map(lambda s: hash_rgb_from_str(s), name_list) )

    # prepare canvas
    reset_plot()
    plt.rcParams['font.size'] = 10
    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 11))

    # plot
    ax.pie(cnt_list, labels=label_list, colors=rgb_list)

    # save fig
    fig_name = get_outfile_name(args, plot_name, args.imgtype)
    plt.savefig(fig_name)
    plt.close()

def plot_waker_waiter_graph(args, csv_data, plot_name, thr_cratio):
    def safe_node_name(s):
        return base64.urlsafe_b64encode(bytes(s, 'UTF-8')).decode("UTF-8")

    g = graphviz.Digraph(comment="thr_cratio = %f" % thr_cratio)

    # csv data: node1, edge, node2, num, ratio, cratio
    #           waker        waiter
    #           [0,    1,    2,     3,   4,     5    ]
    node_set = set()
    # - add edges 
    for node1, edge12, node2, num, ratio, cratio in csv_data.stats:
        if cratio > thr_cratio:
            break
        node_set.add(node1)
        node_set.add(node2)
        g.edge(tail_name=safe_node_name(node1), 
               head_name=safe_node_name(node2), 
               label=" %s (%d)" % (edge12, num))
    # - add nodes with filledcolor
    for node in node_set:
        node_color = hex_color( hash_rgb_from_str(node) )
        g.node(name=safe_node_name(node), label=node, 
               style="filled", fillcolor=node_color, color=node_color)
        
    # - save the graph to file
    odir, fname, suffix = get_outfile_name_x(
            args, plot_name + "_%f_dot" % thr_cratio, args.imgtype)
    g.format= args.imgtype
    g.filename = fname
    g.directory = odir
    try:
        g.render(view=False)
    except Exception as e:
        print(e)

def get_res_path(args, fname):
    afil = os.path.abspath(fname)
    rfil = os.path.relpath(afil, start = os.path.dirname(args.output))
    return afil, rfil

def gen_md_title(title, f):
    print("``` {=html}", file = f)
    print("<style>", file = f)
    print("body { min-width: 80% !important; }", file = f)
    print("</style>", file = f)
    print("```", file = f)

    print("---", file = f)
    print("title: %s" % title, file = f)
    print("date: %s" % datetime.datetime.now(), file = f)
    print("---", file = f)
    print("\n\n", file = f)

def noop(x):
    return x

def gen_md_tbl_figs(args, col_names, fea, header, f, trans_fn = noop):
    # generate header row
    if header:
        l1, l2 = "|", "|"
        for col_name in col_names:
            l1 = l1 + " **" + col_name + "** | "
            l2 = l2 + " ---: | "
        print(l1, file = f)
        print(l2, file = f)

    # figures
    l = "|"
    for col_name in col_names:
        fig_name = get_outfile_name(args, fea + trans_fn(col_name), args.imgtype)
        afil, rfil = get_res_path(args, fig_name)
        img = "![](" + rfil + ")"
        l = l + " " + img + " | "
    print(l, file = f)
    print("\n\n", file = f)

def gen_config(args, md_f):
    # configs
    gen_md_title("%s" % args.prefix , md_f)

    print("### Logs \n", file = md_f)
    afil = os.path.abspath( get_schedmon_log_name(args, "timehist_full") )
    print("- file: %s" % afil, file = md_f)

    sz = os.stat(afil).st_size / (1024.0 * 1024.0)
    print("- size: %.2f MB" % sz, file = md_f)

    mtime = datetime.datetime.fromtimestamp(os.stat(afil).st_mtime).strftime('%Y-%m-%d %H:%M')
    print("- mtime: %s" % mtime, file = md_f)
    print("\n\n", file = md_f)


def gen_task_idle_stat_report(args, results, md_f):
    print("## System-wide task and idle statistics\n\n", file = md_f)
    fname = get_outfile_name(args, "sw_task_stat", "csv")
    afil, rfil = get_res_path(args, fname)
    print("- [System-wide task stats data](%s)\n" % rfil, file = md_f)

    fname = get_outfile_name(args, "tsov_sw_task_stat", args.imgtype)
    afil, rfil = get_res_path(args, fname)
    print("![](" + rfil + ")\n", file = md_f)
    print("\n\n", file = md_f)

    fname = get_outfile_name(args, "sw_idle_stat", "csv")
    afil, rfil = get_res_path(args, fname)
    print("- [System-wide idle stats data](%s)\n" % rfil, file = md_f)

    fname = get_outfile_name(args, "tsov_sw_idle_stat", args.imgtype)
    afil, rfil = get_res_path(args, fname)
    print("![](" + rfil + ")\n", file = md_f)
    print("\n\n", file = md_f)

    print("## Task statistics using per-task average\n\n", file = md_f)
    # links to raw data
    fname = get_outfile_name(args, "task_stat", "csv")
    afil, rfil = get_res_path(args, fname)
    print("- [task stats data](%s)\n" % rfil, file = md_f)

    # figs table
    mx = len(results.task_csv.names[0:-4])
    for i in range(1, len(results.task_csv.names[0:-4]), 8):
        i_next = min(i+8, mx)
        gen_md_tbl_figs(args, results.task_csv.names[i:i_next], 
                        "task_dist_", True, md_f)
        print("\n\n", file = md_f)
    print("\n\n", file = md_f)

def gen_task_idle_stat(args, r, md_f):
    # generate CSV data
    ofname = get_outfile_name(args, "task_stat", "csv")
    with open(ofname, "w") as f:
        gen_stat_in_csv(r.task_csv, f)

    ofname = get_outfile_name(args, "sw_task_stat", "csv")
    with open(ofname, "w") as f:
        gen_stat_in_csv(r.sw_task_csv, f)

    ofname = get_outfile_name(args, "sw_idle_stat", "csv")
    with open(ofname, "w") as f:
        gen_stat_in_csv(r.sw_idle_csv, f)

    # generate distribution graphs for all tasks
    plot_task_dist_figs(args, r.task_csv.names, r.task_csv.stats)

    # generate tsov graphs for system-wide task and idle stat
    plot_per_task_stat_over_time(args, r.sw_task_stat, "sw_task_stat", 
                                 "tsov_sw_task_stat", x_size = 8, y_size = 14)
    plot_per_task_stat_over_time(args, r.sw_idle_stat, "sw_idle_stat", 
                                 "tsov_sw_idle_stat", x_size = 8, y_size = 14)

    # generate markdown
    gen_task_idle_stat_report(args, r, md_f)

def gen_task_stat_corr(args, results, md_f):
    # generate correlation graphs
    for cset in results.corr_sets:
        plot_corr_figs(args, cset.corrs, "task_stat_corr" + cset.name)

    # generate markdown
    print("## Correlation between task features\n\n", file = md_f)
    print("- PCC: Pearson correlation coefficient\n", file = md_f)
    print("- WCC: Weighted correlation coefficient using `num_sched`\n", file = md_f)

    for cset in results.corr_sets:
        print("### - %s\n" % cset.question, file = md_f)
        fig_name = get_outfile_name(args, "task_stat_corr" + cset.name, args.imgtype)
        afil, rfil = get_res_path(args, fig_name)
        print("![](" + rfil + ")\n", file = md_f)
    print("\n\n", file = md_f)


def get_fssafe_task_name(task_name):
    # NOTE: urlsafe_base64 encoding would be safer 
    # but it is hard to know what it is.
    non_safe_chars = [" ", "\t", "\\", "/", "[", "]", "{", "}", "&", 
                      ":", "\n", "\"", "'", "$"]
    for c in non_safe_chars:
        task_name = task_name.replace(c, "_")
    return task_name

def gen_task_stat_over_time(args, results, md_f):
    def get_index_stat_fea(results, fea_name):
        for i, name in enumerate(results.task_csv.names):
            if name == fea_name:
                return i
        return -1

    # generate tsov graphs for all tasks
    task_names = []
    ti = get_index_stat_fea(results, "task_name")
    for ts_r in results.task_csv.stats:
        task_name = ts_r[ti]
        task_name_fssafe = get_fssafe_task_name(task_name)
        task = ev_task.find(task_name)
        task_color = task.color
        task_stat = task.task_stat
        task_names.append(task_name)
        plot_per_task_stat_over_time(args, task_stat, task_name_fssafe, 
                                     "tsov_" + task_name_fssafe)

    # intergrate the results into the report
    print("## Task's wait_time, sched_delay, and runtimes\n\n", file = md_f)
    for i in range(0, len(task_names), 6):
        gen_md_tbl_figs(args, task_names[i:i+6], "tsov_", True, 
                        md_f, trans_fn = get_fssafe_task_name)
        print("\n\n", file = md_f)
    print("\n\n", file = md_f)

def gen_callstack_stat(args, r, md_f):
    # generate CSV data for raw callstacks
    ofname = get_outfile_name(args, "callstack_stat", "csv")
    with open(ofname, "w") as f:
        gen_stat_in_csv(r.cs_csv, f)

    # generate CSV data for callstack types
    ofname = get_outfile_name(args, "callstack_type_stat", "csv")
    with open(ofname, "w") as f:
        gen_stat_in_csv(r.cst_csv, f)

    # plot callstack types
    trans_stats = list(zip(*r.cst_csv.stats))
    name_list = trans_stats[0]
    cnt_list = trans_stats[1]
    plot_pie_chart(args, name_list, cnt_list, "callstack_type_stat")

    # generate markdown
    print("## Callstacks tiggered scheduling\n\n", file = md_f)
    ofname = get_outfile_name(args, "callstack_stat", "csv")
    afil, rfil = get_res_path(args, ofname)
    print("- [callstacks](" + rfil + ")", file = md_f)

    ofname = get_outfile_name(args, "callstack_type_stat", "csv")
    afil, rfil = get_res_path(args, ofname)
    print("- [callstack types](" + rfil + ")", file = md_f)
    print("\n\n", file = md_f)

    ofname = get_outfile_name(args, "callstack_type_stat", args.imgtype)
    afil, rfil = get_res_path(args, ofname)
    print("![](" + rfil + ")", file = md_f)
    print("\n\n", file = md_f)

def gen_waiter_waker_stat(args, results, md_f):
    r = results

    # generate the detailed data in CSV
    csv_configs = [("wcwn", r.wcw_csv, "Waker-callstack-waiter => count"),
                   ("wtwn", r.wtw_csv, "Waker-callstack type-waiter => count"),
                   ("w_cwn", r.w_cwn_csv, "Waker => callstack-waiter => count"),
                   ("w_twn", r.w_twn_csv, "Waker => callstack type-wakter => count"),
                   ("wc_wn", r.wc_wn_csv, "Waiter => callstack-waker => count"),
                   ("wt_wn", r.wt_wn_csv, "Waiter => callstack type-waker => count"),
                   ("waiter_task", r.waiter_csv, "Waiter count"), 
                   ("waker_task", r.waker_csv, "Waker count"),
                   ("wcw_cs", r.wcw_cs_csv, "Callstack count"), 
                   ("wtw_cst", r.wtw_cst_csv, "Callstack type count")]
    for name, data, desc in csv_configs:
        ofname = get_outfile_name(args, name, "csv")
        with open(ofname, "w") as f:
            gen_stat_in_csv(data, f)

    # plot overview data in pie charts
    img_configs1 = [("waiter_task", r.waiter_csv.stats, "Distribution of waiters"),
                   ("waker_task", r.waker_csv.stats, "Discribution of wakers"),
                   ("wtw_cst", r.wtw_cst_csv.stats, "Distribution of waiting callstack types")]
    for name, data, desc in img_configs1:
        trans_stats = list(zip(*data))
        plot_pie_chart(args, trans_stats[0], trans_stats[1], name)

    img_configs2 = [("wcw_cs", r.wcw_cs_csv.stats, "Distribution of waiting callstacks"),]
    for name, data, desc in img_configs2:
        trans_stats = list(zip(*data))
        plot_pie_chart(args, trans_stats[0], trans_stats[2], name)

    # plot graphs
    global wcw_thrs
    for t in wcw_thrs:
        __start_time = datetime.datetime.now()
        plot_waker_waiter_graph(args, r.wtw_csv, "wtw_graph", t)
        __end_time = datetime.datetime.now()
        time_diff = (__end_time - __start_time).total_seconds()
        if time_diff >= args.timelimit:
            break

    # generate markdown
    print("## Waker-waiter analysis\n\n", file = md_f)
    for name, data, desc in csv_configs:
        ofname = get_outfile_name(args, name, "csv")
        afil, rfil = get_res_path(args, ofname)
        print("- [" + desc +" ](" + rfil + ")\n", file = md_f)
    print("\n\n", file = md_f)

    img_configs = img_configs1 + img_configs2
    for name, data, desc in img_configs:
        ofname = get_outfile_name(args, name, args.imgtype)
        afil, rfil = get_res_path(args, ofname)
        print("### - " + desc +"\n", file = md_f)
        print("![](" + rfil + ")\n", file = md_f)
    print("\n\n", file = md_f)

    for t in wcw_thrs:
        ofname = get_outfile_name(args, "wtw_graph_%f_dot" % t, args.imgtype)
        if os.path.isfile(ofname):
            afil, rfil = get_res_path(args, ofname)
            print("### - %.2f percentile of waker-waiter\n" % t, file = md_f)
            print("![](" + rfil + ")\n", file = md_f)
    print("\n\n", file = md_f)


def analyze_stats_correlation(args, results):
    def get_stat(key, stat_names, stat_cols):
        for stat_name, stat in zip(stat_names, stat_cols):
            if stat_name == key:
                return stat
        return None

    # list of stat-pairs
    results.corr_sets = [
            corr_set("_num_sched",
                     "How userful is num_sched in predicting task's behavior?",
                     [pearson_corr("num_sched", "num_mig", "task_name", "num_sched"), 
                      pearson_corr("num_sched", "runtime_avg", "task_name", "num_sched")]),
            corr_set("_runtime_total",
                     "How useful is the vruntime in predicting task's behavior?",
                     [pearson_corr("runtime_total", "num_sched", "task_name", "num_sched"),
                      pearson_corr("runtime_total", "runtime_avg", "task_name", "num_sched")]),
            corr_set("_wake_wait",
                     "How useful is the num_wake/wait in predicting task's behavior?",
                     [pearson_corr("num_wake", "num_sched", "task_name", "num_sched"),
                      pearson_corr("num_wait", "num_sched", "task_name", "num_sched"),
                      pearson_corr("num_wake", "runtime_avg", "task_name", "num_sched"),
                      pearson_corr("num_wait", "runtime_avg", "task_name", "num_sched")]),
            corr_set("_runtime_avg_aad",
                     "Is task's runtime stable and predictable using average?",
                     [pearson_corr("runtime_avg", "runtime_aad", "task_name", "num_sched")]),
            corr_set("_wait_time",
                     "Is task's wait time an useful indicator for something?",
                     [pearson_corr("num_sched", "wait_time_total", "task_name", "num_sched"),
                      pearson_corr("runtime_avg", "wait_time_total", "task_name", "num_sched"),
                      pearson_corr("num_sched", "wait_time_avg", "task_name", "num_sched"), 
                      pearson_corr("runtime_avg", "wait_time_avg", "task_name", "num_sched"),
                      pearson_corr("wait_time_avg", "wait_time_aad", "task_name", "num_sched")]),
            corr_set("_sched_delay",
                     "Can we predict scheding delay?",
                     [pearson_corr("num_sched", "sched_delay_avg", "task_name", "num_sched"),
                      pearson_corr("runtime_avg", "sched_delay_avg", "task_name", "num_sched"),
                      pearson_corr("runtime_total", "sched_delay_avg", "task_name", "num_sched")]),
            ]

    # calc pearson correlations
    stat_cols = transpose_2d_list(results.task_csv.stats)
    for crset in results.corr_sets:
        for c in crset.corrs:
            c.x_data = get_stat(c.x_label, results.task_csv.names, stat_cols)
            c.y_data = get_stat(c.y_label, results.task_csv.names, stat_cols)
            c.data_names = get_stat(c.data_label, results.task_csv.names, stat_cols)
            c.w_data = get_stat(c.w_label, results.task_csv.names, stat_cols)
            c.calc_pcc()
            c.calc_wcc()

def analyze_callstack(args, r):
    ev_callstack.analyze_stat()

    # collect call stack information
    stat_names = ["callstack", "callstack_type", "num_sched", "num_task_scheduled_out"]
    stat_formats = ["{0:^50}", "{0:^50}"] + ["{0:>14}"] * (len(stat_names) - 1)
    stats = []
    for k, cs in ev_callstack.cs_dict.items():
        s = cs.callstack_stat
        stats.append( (str(cs), cs.cs_type, s.num_sched_events, s.num_ctask) )

    # sort by the number of events
    stats.sort(reverse = True, key = lambda x: x[1])

    # return the results
    r.cs_csv.names = stat_names
    r.cs_csv.formats = stat_formats
    r.cs_csv.stats = stats

def analyze_callstack_type(args, r):
    # aggregate cs state results with the same cs type
    stat_names = ["callstack_type", "num_sched", "num_task_scheduled_out"]
    stat_formats = ["{0:^50}"] + ["{0:>14}"] * (len(stat_names) - 1)
    stats = []

    for k, cst in ev_callstack.cst_dict.items():
        ctask_set = set()
        num_sched_events = 0
        for cs in cst:
            ctask_set = ctask_set.union(cs.cs_ctasks)
            num_sched_events = num_sched_events + cs.callstack_stat.num_sched_events
        num_ctask = len(ctask_set)
        stats.append( (k, num_sched_events, num_ctask) )

    # sort by the number of events
    stats.sort(reverse = True, key = lambda x: x[1])

    # return the results
    r.cst_csv.names = stat_names
    r.cst_csv.formats = stat_formats
    r.cst_csv.stats = stats
    

def process_awake_event(r):
    def find_waiting_event(t_waiter, wake_at, events):
        for i in reversed(range(wake_at)):
            e = events[i]
            if e.ty == ev_type.SCHED and e.cur_task == t_waiter:
                return e
        return None

    # process awake events
    events = list(r.sched_events.values())
    for i, e in enumerate(events):
        if e.ty != ev_type.AWAKE:
            continue
        # find waker and waiter
        t_waker  = e.next_task   # current task  (i.e., waker)
        t_waiter = e.cur_task    # awakened task (i.e., waiter)

        # find waiter's waiting call stack
        wait_event = find_waiting_event(t_waiter, i, events)
        if wait_event == None:
            # the earlier part of the scheduling events would not be logged
            continue
        waiter_cs = wait_event.callstack

        # add a wake relationship
        task_waker_waiter.add_wcw(t_waker, waiter_cs, t_waiter)

    # update task_stat for wake_num and wait_num
    for tname, cnt in task_waker_waiter.get_waiter():
        t = ev_task.find(tname)
        t.task_stat.wait_num = cnt
    for tname, cnt in task_waker_waiter.get_waker():
        t = ev_task.find(tname)
        t.task_stat.wake_num = cnt

def append_ratio_n_cratio(stats):
    # "waker_task", "waiter_callstack", "waiter_task"
    # "num_wake", "wake_ratio", "wake_cratio"
    col_l= list(zip(*stats))
    t_ll, num_l = col_l[0:-1], col_l[-1]

    ratio_l, cratio_l = [], []
    total_num = sum(num_l) 
    cratio = 0.0
    for num in num_l:
        ratio = 100.0 * (num/total_num)
        cratio = cratio + ratio
        ratio_l.append(ratio)
        cratio_l.append(cratio)

    return list(zip(*t_ll, num_l, ratio_l, cratio_l))

def analyze_waker_waiter(args, r):
    process_awake_event(r)

    # get waker-cs-waiter stats
    # - (waker, callstack, waiter)     => count
    stats = list(task_waker_waiter.get_wcwn())
    r.wcw_csv.stats = append_ratio_n_cratio(stats)
    r.wcw_csv.names = ["waker_task", "waiter_callstack", "waiter_callstack_type",
                       "waiter_task", "num_wake", "wake_ratio", "wake_cratio"]
    r.wcw_csv.formats = ["{0:^30}", "{0:^50}", "{0:^50}", "{0:^30}",
                         "{0:>14}", "{0:>14}", "{0:>14}"]

    # get waker-cst-waiter stats
    # - (waiter, callstak type, waker) => count
    stats = list(task_waker_waiter.get_wtwn())
    r.wtw_csv.stats = append_ratio_n_cratio(stats)
    r.wtw_csv.names = ["waker_task", "waiter_callstack_type", "waiter_task",
                       "num_wake", "wake_ratio", "wake_cratio"]
    r.wtw_csv.formats = r.wcw_csv.formats

    # waker => cs-watier, waker => cst-watier
    tasks = list(task_waker_waiter.get_waker())
    for task, cnt in tasks:
        stats = list(task_waker_waiter.get_w_cwn(str(task)))
        r.w_cwn_csv.stats = r.w_cwn_csv.stats + append_ratio_n_cratio(stats)

        stats = list(task_waker_waiter.get_w_twn(str(task)))
        r.w_twn_csv.stats = r.w_twn_csv.stats + append_ratio_n_cratio(stats)

    r.w_cwn_csv.names = ["waker_task", "waiter_callstack", "waiter_callstack_type",
                         "waiter_task", "num_wake", "wake_ratio"]
    r.w_cwn_csv.formats = ["{0:^30}", "{0:^50}", "{0:^50}", 
                           "{0:^30}", "{0:>14}", "{0:>14}"]
    r.w_twn_csv.names = ["waker_task", "waiter_callstack_type", "waiter_task", 
                         "num_wake", "wake_ratio"]
    r.w_twn_csv.formats = ["{0:^30}", "{0:^50}", "{0:^30}",
                           "{0:>14}", "{0:>14}"]

    # waiter => cs-watier, waker => cst-waker
    tasks = list(task_waker_waiter.get_waiter())
    for task, cnt in tasks:
        stats = list(task_waker_waiter.get_wc_wn(str(task)))
        r.wc_wn_csv.stats = r.wc_wn_csv.stats + append_ratio_n_cratio(stats)

        stats = list(task_waker_waiter.get_wt_wn(str(task)))
        r.wt_wn_csv.stats = r.wt_wn_csv.stats + append_ratio_n_cratio(stats)

    r.wc_wn_csv.names = ["waiter_task", "waiter_callstack", "waiter_callstack_type",
                         "waker_task", "num_wake", "wake_ratio"]
    r.wc_wn_csv.formats = ["{0:^30}", "{0:^50}", "{0:^50}", "{0:^30}",
                           "{0:>14}", "{0:>14}"]
    r.wt_wn_csv.names = ["waiter_task", "waiter_callstack_type", "waker_task",
                         "num_wake", "wake_ratio"]
    r.wt_wn_csv.formats = ["{0:^30}", "{0:^50}", "{0:^30}",
                           "{0:>14}", "{0:>14}"]

    # get waiter stats
    # - waiter                         => count
    stats = list(task_waker_waiter.get_waiter())
    r.waiter_csv.stats = append_ratio_n_cratio(stats)
    r.waiter_csv.names = ["waiter_task", "num_wait", "wake_ratio", "wake_cratio"]
    r.waiter_csv.formats = ["{0:^30}", "{0:>14}", "{0:>14}", "{0:>14}"]

    # get waker stats
    # - waker                          => count
    stats = list(task_waker_waiter.get_waker())
    r.waker_csv.stats = append_ratio_n_cratio(stats)
    r.waker_csv.names = ["waker_task", "num_wake", "wake_ratio", "wake_cratio"]
    r.waker_csv.formats = r.waiter_csv.formats

    # get cs stats
    # - cs                             => count
    stats = list(task_waker_waiter.get_cs())
    r.wcw_cs_csv.stats = append_ratio_n_cratio(stats)
    r.wcw_cs_csv.names = ["callstack", "callstack_type", 
                          "num_wake", "wake_ratio", "wake_cratio"]
    r.wcw_cs_csv. formats = ["{0:^50}", "{0:^50}", 
                             "{0:>14}", "{0:>14}", "{0:>14}"]
  
    # get cst stats
    # - cst                            => count
    stats = list(task_waker_waiter.get_cst())
    r.wtw_cst_csv.stats= append_ratio_n_cratio(stats)
    r.wtw_cst_csv.names = ["callstack_type", "num_wake", "wake_ratio", "wake_cratio"]
    r.wtw_cst_csv.formats = ["{0:^50}", "{0:>14}", "{0:>14}", "{0:>14}"]

def parse_and_analyze_event(args):
    # parse the log
    results = analysis_results()
    parse_schedmon_timehist(args, results)

    # then analyze it
    # 1. 
    analyze_waker_waiter(args, results)
    # 2.
    analyze_task_idle_stat(args, results)
    # 3. 
    analyze_stats_correlation(args, results)
    # 4.
    analyze_callstack(args, results)
    analyze_callstack_type(args, results)

    return results

def gen_report(args, results):
    # generate a report in markdow
    md_fname = args.output + ".md"
    with open(md_fname, "w") as md_f:
        gen_config(args, md_f)
        gen_task_idle_stat(args, results, md_f)
        gen_task_stat_corr(args, results, md_f)
        gen_task_stat_over_time(args, results, md_f)
        gen_callstack_stat(args, results, md_f)
        gen_waiter_waker_stat(args, results, md_f)

    # convert the markdown to html
    html_fname = args.output + ".html"
    cmd = "pandoc --standalone --toc %s -o %s" % (md_fname, html_fname)
    p = subprocess.Popen(cmd, shell=True, stdout=None, stderr=None)
    p.wait()

def get_cmd_options(argv):
    parser = argparse.ArgumentParser(
            prog = "schedinsight",
            description = "Report the detailed analysis of scheduliing activities collected by `perf sched record`",)
    parser.add_argument('-l', '--logdir', action='store', required=True,
                        help='a log directory') 
    parser.add_argument('-p', '--prefix', action='store', required=True,
                        help='log file prefix') 
    parser.add_argument('-o', '--output', action='store', required=True,
                        help='a target report file name in markdown format') 
    parser.add_argument('-i', '--imgtype', action='store', default="png",
                        help='type of image format (png, svg)' ) 
    parser.add_argument('-k', '--pickle', action='store_true',
                        help='use pickle whenever possible' ) 
    parser.add_argument('-s', '--minsched', action='store', type=int, default=100,
                        help='set the minimum number of schedules for task analysis' ) 
    parser.add_argument('-t', '--timelimit', action='store', type=int, default=400,
                        help='time limit to draw a graph in seconds' ) 
    args = parser.parse_args(argv)

    return args

if __name__ == "__main__":
    sys.setrecursionlimit(100000)    # extend recursion limit for huge logs

    args = get_cmd_options(sys.argv[1:])

    results = parse_and_analyze_event(args)
    gen_report(args, results)


