@article{KoumarelasJiangNaumann2020,
  author    = {Koumarelas, Ioannis and Jiang, Lan and Naumann, Felix},
  title     = {Data preparation for duplicate detection},
  series = {Journal of data and information quality : (JDIQ)},
  volume    = {12},
  journal   = {Journal of data and information quality : (JDIQ)},
  number    = {3},
  publisher = {Association for Computing Machinery},
  address   = {New York},
  issn      = {1936-1955},
  doi       = {10.1145/3377878},
  pages     = {24},
  year      = {2020},
  abstract  = {Data errors represent a major issue in most application workflows. Before any important task can take place, a certain data quality has to be guaranteed by eliminating a number of different errors that may appear in data. Typically, most of these errors are fixed with data preparation methods, such as whitespace removal. However, the particular error of duplicate records, where multiple records refer to the same entity, is usually eliminated independently with specialized techniques. Our work is the first to bring these two areas together by applying data preparation operations under a systematic approach prior to performing duplicate detection. <br /> Our process workflow can be summarized as follows: It begins with the user providing as input a sample of the gold standard, the actual dataset, and optionally some constraints to domain-specific data preparations, such as address normalization. The preparation selection operates in two consecutive phases. First, to vastly reduce the search space of ineffective data preparations, decisions are made based on the improvement or worsening of pair similarities. Second, using the remaining data preparations an iterative leave-one-out classification process removes preparations one by one and determines the redundant preparations based on the achieved area under the precision-recall curve (AUC-PR). Using this workflow, we manage to improve the results of duplicate detection up to 19\% in AUC-PR.},
  language  = {en}
}
@article{JiangNaumann2020,
  author    = {Jiang, Lan and Naumann, Felix},
  title     = {Holistic primary key and foreign key detection},
  series = {Journal of intelligent information systems : JIIS},
  volume    = {54},
  journal   = {Journal of intelligent information systems : JIIS},
  number    = {3},
  publisher = {Springer},
  address   = {Dordrecht},
  issn      = {0925-9902},
  doi       = {10.1007/s10844-019-00562-z},
  pages     = {439 -- 461},
  year      = {2020},
  abstract  = {Primary keys (PKs) and foreign keys (FKs) are important elements of relational schemata in various applications, such as query optimization and data integration. However, in many cases, these constraints are unknown or not documented. Detecting them manually is time-consuming and even infeasible in large-scale datasets. We study the problem of discovering primary keys and foreign keys automatically and propose an algorithm to detect both, namely Holistic Primary Key and Foreign Key Detection (HoPF). PKs and FKs are subsets of the sets of unique column combinations (UCCs) and inclusion dependencies (INDs), respectively, for which efficient discovery algorithms are known. Using score functions, our approach is able to effectively extract the true PKs and FKs from the vast sets of valid UCCs and INDs. Several pruning rules are employed to speed up the procedure. We evaluate precision and recall on three benchmarks and two real-world datasets. The results show that our method is able to retrieve on average 88\% of all primary keys, and 91\% of all foreign keys. We compare the performance of HoPF with two baseline approaches that both assume the existence of primary keys.},
  language  = {en}
}
@article{VitaglianoJiangNaumann2021,
  author    = {Vitagliano, Gerardo and Jiang, Lan and Naumann, Felix},
  title     = {Detecting layout templates in complex multiregion files},
  series = {Proceedings of the VLDB Endowment},
  volume    = {15},
  journal   = {Proceedings of the VLDB Endowment},
  number    = {3},
  publisher = {Association for Computing Machinery},
  address   = {New York},
  issn      = {2150-8097},
  doi       = {10.14778/3494124.3494145},
  pages     = {646 -- 658},
  year      = {2021},
  abstract  = {Spreadsheets are among the most commonly used file formats for data management, distribution, and analysis. Their widespread employment makes it easy to gather large collections of data, but their flexible canvas-based structure makes automated analysis difficult without heavy preparation. One of the common problems that practitioners face is the presence of multiple, independent regions in a single spreadsheet, possibly separated by repeated empty cells. We define such files as "multiregion" files. In collections of various spreadsheets, we can observe that some share the same layout. We present the Mondrian approach to automatically identify layout templates across multiple files and systematically extract the corresponding regions. Our approach is composed of three phases: first, each file is rendered as an image and inspected for elements that could form regions; then, using a clustering algorithm, the identified elements are grouped to form regions; finally, every file layout is represented as a graph and compared with others to find layout templates. We compare our method to state-of-the-art table recognition algorithms on two corpora of real-world enterprise spreadsheets. Our approach shows the best performances in detecting reliable region boundaries within each file and can correctly identify recurring layouts across files.},
  language  = {en}
}