利用递归思想处理半结构化数据¶

[{
    'state': 'Florida',
    'shortname': 'FL',
    'info': {
        'governor': 'Rick Scott'
    },
    'counties': [
        {'name': 'Dade', 'population': 12345},
        {'name': 'Broward', 'population': 40000},
        {'name': 'Palm Beach', 'population': 60000}
    ]},
    {
    'state': 'Ohio',
    'shortname': 'OH',
    'info': {
        'governor': 'John Kasich'
    },
    'counties': [
        {'name': 'Summit', 'population': 1234},
        {'name': 'Cuyahoga', 'population': 1337}]
    }]

# 尝试直接转化为DataFrame
df = pd.DataFrame(data)
print(df)

# 使用json_normalize():
pd.json_normalize(
    data,
    record_path = 'counties',  # 定义数据粒度
    meta = ['state', 'shortname',['info', 'governor']] # 定义存入结果表的列名
    )

def Fibonacci(n):
    def _recurse(n):
        if n == 0: # 边界条件
            return 0
        elif n == 1: # 边界条件
            return 1
        else:
            return _recurse(n-1) + _recurse(n-2) # 调用自身，缩小问题规模
    return _recurse(n)

def _json_normalize(
    data: Union[Dict, List[Dict]],
    record_path: Optional[Union[str, List]] = None,
    meta: Optional[Union[str, List[Union[str, List[str]]]]] = None,
    meta_prefix: Optional[str] = None,
    record_prefix: Optional[str] = None,
    errors: str = "raise",
    sep: str = ".",
    max_level: Optional[int] = None,
)
    # 定义一些辅助函数
    def _pull_field(js: Dict[str, Any], spec: Union[List, str])：
        """Internal function to pull field"""
        # 省略具体实现代码
        ...
        return result

    def _pull_records(js: Dict[str, Any], spec: Union[List, str]) -> List:
        """
        Internal function to pull field for records, and similar to
        _pull_field, but require to return list.
        """
        # 省略具体实现代码
        ...
        return result  

    # 省略一些对输入参数的处理和判定的代码   

    # 定义一些函数内变量
    _meta = [m if isinstance(m, list) else [m] for m in meta] # 需要展现在结果里的字段
    records: List = []
    lengths = []
    # meta_val用于存储fields对应的数值
    meta_vals: DefaultDict = defaultdict(list)
    # 在深层字典里的字段，用上一层的字段名+分隔符+这一层的字段名代替，防止字段重复，如例子中的 ['info', 'governor'] 处理成 info.governor
    meta_keys = [sep.join(val) for val in _meta]

    # 核心代码：
    # Disastrously inefficient for now
    def _recursive_extract(data, path, seen_meta, level=0):
        if isinstance(data, dict):
            data = [data]

        if len(path) > 1: #这里对于多个record_path的情况进行递归
            for obj in data:
                for val, key in zip(_meta, meta_keys):
                    if level + 1 == len(val):
                        seen_meta[key] = _pull_field(obj, val[-1])

                _recursive_extract(obj[path[0]], path[1:], seen_meta, level=level + 1)

        else:
            # 遍历当前层的data
            for obj in data:
                # 取出当前层所有的records (list of dict)
                recs = _pull_records(obj, path[0])
                # pandas 中的另一个内置函数，如果records里面有dict,将dict打开。
                # 例如如果有一条记录是{'a':{'b':1}} 则会变成 {'a.b':1}
                recs = [
                    nested_to_record(r, sep=sep, max_level=max_level)
                    if isinstance(r, dict)
                    else r
                    for r in recs
                ]

                # For repeating the metadata later
                lengths.append(len(recs)) # 记录当前层级records的数量
                for val, key in zip(_meta, meta_keys):
                    if level + 1 > len(val): # 如果字段在下一层dict，如val为['info', 'governor']，则取seen_meta['info.governer']
                        meta_val = seen_meta[key]
                    else: # 取出当前层字段对应的value
                        try:
                            meta_val = _pull_field(obj, val[level:])
                        except KeyError as e:
                            if errors == "ignore":
                                meta_val = np.nan
                            else:
                                raise KeyError(
                                    "Try running with errors='ignore' as key "
                                    f"{e} is not always present"
                                ) from e
                    meta_vals[key].append(meta_val)
                records.extend(recs)

    _recursive_extract(data, record_path, {}, level=0)

    # 省略一些对result的格式后处理代码

    return result