Skip to content

Excel layout kg

ExcelLayoutKG

Layout Knowledge Graph For each excel, each sheet will be a node (same a page in pdf) Then we will have the images and tables connect to the sheet, also the summary and description will be added to the sheet node

Source code in Docs2KG/kg/excel_layout_kg.py 
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
class ExcelLayoutKG:
    """
    Layout Knowledge Graph
    For each excel, each sheet will be a node (same a page in pdf)
    Then we will have the images and tables connect to the sheet, also the
    summary and description will be added to the sheet node

    """

    def __init__(
        self,
        folder_path: Path,
        input_format: str = "pdf_exported",
    ):
        """
        Initialize the class with the pdf file
        The goal of this is to construct the layout knowledge graph

        Args:
            folder_path (Path): The path to the pdf file

        """
        self.folder_path = folder_path
        self.kg_folder = self.folder_path / "kg"
        if not self.kg_folder.exists():
            self.kg_folder.mkdir(parents=True, exist_ok=True)
        self.kg_json = {}
        self.metadata = json.load((self.folder_path / "metadata.json").open())
        self.sentence_transformer = SentenceTransformer("all-MiniLM-L6-v2")
        self.input_format = input_format

    def create_kg(self):
        """
        Create the layout knowledge graph
        """
        self.document_kg()

    def document_kg(self):
        """
        Construct the layout knowledge graph skeleton first

        We will require the md.json.csv file with the following columns:

        - layout_json
        """
        # 1. add the document node

        self.kg_json = {
            "node_type": "excel",
            "uuid": str(uuid4()),
            "node_properties": self.metadata,
            "children": [],
        }

        # 2. add page nodes
        pages_json = []
        text_folder = self.folder_path / "texts"
        md_json_csv = text_folder / "md.json.csv"
        summary_and_desc_df = pd.read_csv(md_json_csv)
        columns = summary_and_desc_df.columns.tolist()
        logger.debug(f"Columns: {columns}")
        # we will focus on the layout json
        table_df = pd.read_csv(self.folder_path / "tables" / "tables.csv")
        image_df = pd.read_csv(self.folder_path / "images" / "images.csv")

        for index, row in summary_and_desc_df.iterrows():
            logger.info(f"Processing page_index {index}")
            try:
                # get table_item
                table_item = table_df[table_df["page_index"] == index]
                image_item = image_df[image_df["page_index"] == index]
                page_json = {
                    "node_type": "page",
                    "uuid": str(uuid4()),
                    "node_properties": {
                        "page_number": row["page_number"],
                        "page_text": row["text"],
                        "sheet_name": row["sheet_name"],
                        "summary": row["summary"] if pd.notna(row["summary"]) else "",
                        "content": row["desc"] if pd.notna(row["desc"]) else "",
                    },
                    "children": [
                        # we will have image node, table node
                        {
                            "node_type": "table_csv",
                            "uuid": str(uuid4()),
                            "node_properties": {
                                "table_path": table_item["file_path"].values[0],
                                "table_index": int(table_item["table_index"].values[0]),
                            },
                            "children": [],
                        },
                        {
                            "node_type": "image",
                            "uuid": str(uuid4()),
                            "node_properties": {
                                "image_path": image_item["file_path"].values[0],
                                "filename": image_item["filename"].values[0],
                            },
                            "children": [],
                        },
                    ],
                }
                pages_json.append(page_json)
            except Exception as e:
                logger.error(f"Error in row {index}: {e}")

                logger.exception(e)
                # if this is an unhandled error
                # we should still keep all data for this page, so we will construct a page with everything we have
                page_json = {
                    "node_type": "page",
                    "uuid": str(uuid4()),
                    "node_properties": {
                        "page_number": row["page_number"],
                        "page_text": row["text"],
                        "sheet_name": row["sheet_name"],
                        "summary": row["summary"] if pd.notna(row["summary"]) else "",
                        "content": row["desc"] if pd.notna(row["desc"]) else "",
                    },
                    "children": [],
                }
                pages_json.append(page_json)

        self.kg_json["children"] = pages_json
        self.export_kg()

    def export_kg(self) -> None:
        """
        Export the knowledge graph to json file
        """
        with open(self.kg_folder / "layout_kg.json", "w") as f:
            json.dump(self.kg_json, f, indent=2)

    def load_kg(self):
        """
        Load the knowledge graph from JSON
        """
        with open(self.kg_folder / "layout_kg.json", "r") as f:
            self.kg_json = json.load(f)

__init__(folder_path, input_format='pdf_exported')

Initialize the class with the pdf file The goal of this is to construct the layout knowledge graph

Parameters:

Name Type Description Default
folder_path Path

The path to the pdf file

 required
Source code in Docs2KG/kg/excel_layout_kg.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
def __init__(
    self,
    folder_path: Path,
    input_format: str = "pdf_exported",
):
    """
    Initialize the class with the pdf file
    The goal of this is to construct the layout knowledge graph

    Args:
        folder_path (Path): The path to the pdf file

    """
    self.folder_path = folder_path
    self.kg_folder = self.folder_path / "kg"
    if not self.kg_folder.exists():
        self.kg_folder.mkdir(parents=True, exist_ok=True)
    self.kg_json = {}
    self.metadata = json.load((self.folder_path / "metadata.json").open())
    self.sentence_transformer = SentenceTransformer("all-MiniLM-L6-v2")
    self.input_format = input_format

create_kg()

Create the layout knowledge graph

Source code in Docs2KG/kg/excel_layout_kg.py 
44
45
46
47
48
def create_kg(self):
    """
    Create the layout knowledge graph
    """
    self.document_kg()

document_kg()

Construct the layout knowledge graph skeleton first

We will require the md.json.csv file with the following columns:

  • layout_json
Source code in Docs2KG/kg/excel_layout_kg.py 
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def document_kg(self):
    """
    Construct the layout knowledge graph skeleton first

    We will require the md.json.csv file with the following columns:

    - layout_json
    """
    # 1. add the document node

    self.kg_json = {
        "node_type": "excel",
        "uuid": str(uuid4()),
        "node_properties": self.metadata,
        "children": [],
    }

    # 2. add page nodes
    pages_json = []
    text_folder = self.folder_path / "texts"
    md_json_csv = text_folder / "md.json.csv"
    summary_and_desc_df = pd.read_csv(md_json_csv)
    columns = summary_and_desc_df.columns.tolist()
    logger.debug(f"Columns: {columns}")
    # we will focus on the layout json
    table_df = pd.read_csv(self.folder_path / "tables" / "tables.csv")
    image_df = pd.read_csv(self.folder_path / "images" / "images.csv")

    for index, row in summary_and_desc_df.iterrows():
        logger.info(f"Processing page_index {index}")
        try:
            # get table_item
            table_item = table_df[table_df["page_index"] == index]
            image_item = image_df[image_df["page_index"] == index]
            page_json = {
                "node_type": "page",
                "uuid": str(uuid4()),
                "node_properties": {
                    "page_number": row["page_number"],
                    "page_text": row["text"],
                    "sheet_name": row["sheet_name"],
                    "summary": row["summary"] if pd.notna(row["summary"]) else "",
                    "content": row["desc"] if pd.notna(row["desc"]) else "",
                },
                "children": [
                    # we will have image node, table node
                    {
                        "node_type": "table_csv",
                        "uuid": str(uuid4()),
                        "node_properties": {
                            "table_path": table_item["file_path"].values[0],
                            "table_index": int(table_item["table_index"].values[0]),
                        },
                        "children": [],
                    },
                    {
                        "node_type": "image",
                        "uuid": str(uuid4()),
                        "node_properties": {
                            "image_path": image_item["file_path"].values[0],
                            "filename": image_item["filename"].values[0],
                        },
                        "children": [],
                    },
                ],
            }
            pages_json.append(page_json)
        except Exception as e:
            logger.error(f"Error in row {index}: {e}")

            logger.exception(e)
            # if this is an unhandled error
            # we should still keep all data for this page, so we will construct a page with everything we have
            page_json = {
                "node_type": "page",
                "uuid": str(uuid4()),
                "node_properties": {
                    "page_number": row["page_number"],
                    "page_text": row["text"],
                    "sheet_name": row["sheet_name"],
                    "summary": row["summary"] if pd.notna(row["summary"]) else "",
                    "content": row["desc"] if pd.notna(row["desc"]) else "",
                },
                "children": [],
            }
            pages_json.append(page_json)

    self.kg_json["children"] = pages_json
    self.export_kg()

export_kg()

Export the knowledge graph to json file

Source code in Docs2KG/kg/excel_layout_kg.py 
140
141
142
143
144
145
def export_kg(self) -> None:
    """
    Export the knowledge graph to json file
    """
    with open(self.kg_folder / "layout_kg.json", "w") as f:
        json.dump(self.kg_json, f, indent=2)

load_kg()

Load the knowledge graph from JSON

Source code in Docs2KG/kg/excel_layout_kg.py 
147
148
149
150
151
152
def load_kg(self):
    """
    Load the knowledge graph from JSON
    """
    with open(self.kg_folder / "layout_kg.json", "r") as f:
        self.kg_json = json.load(f)