Greg Pauloski

@inproceedings{pauloski2024taps,
    author = {Pauloski, J. Gregory and Hayot-Sasson, Valerie and Gonthier, Maxime and Hudson, Nathaniel and Pan, Haochen and Zhou, Sicheng and Foster, Ian and Chard, Kyle},
    title = {{TaPS: A Performance Evaluation Suite for Task-based Execution Frameworks}},
    address = {New York, NY, USA},
    booktitle = {IEEE 20th International Conference on e-Science},
    doi = {10.1109/e-Science62913.2024.10678702},
    pages = {1-10},
    publisher = {IEEE},
    year = {2024}
}

@misc{pauloski2024proxystore,
    author = {J. Gregory Pauloski and Valerie Hayot-Sasson and Logan Ward and Alexander Brace and André Bauer and Kyle Chard and Ian Foster},
    title = {{Object Proxy Patterns for Accelerating Distributed Applications}},
    archiveprefix = {arXiv},
    eprint = {2407.01764},
    primaryclass = {cs.DC},
    url = {https://arxiv.org/abs/2407.01764},
    year = {2024}
}

@inproceedings{pauloski2023proxystore,
    author = {Pauloski, J. Gregory and Hayot-Sasson, Valerie and Ward, Logan and Hudson, Nathaniel and Sabino, Charlie and Baughman, Matt and Chard, Kyle and Foster, Ian},
    title = {{Accelerating Communications in Federated Applications with Transparent Object Proxies}},
    address = {New York, NY, USA},
    articleno = {59},
    booktitle = {Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis},
    doi = {10.1145/3581784.3607047},
    isbn = {9798400701092},
    location = {Denver, CO, USA},
    numpages = {15},
    publisher = {Association for Computing Machinery},
    series = {SC '23},
    url = {https://doi.org/10.1145/3581784.3607047},
    year = {2023}
}

@misc{hudson2024flight,
    author = {Nathaniel Hudson and Valerie Hayot-Sasson and Yadu Babuji and Matt Baughman and J. Gregory Pauloski and Ryan Chard and Ian Foster and Kyle Chard},
    title = {{Flight: A FaaS-Based Framework for Complex and Hierarchical Federated Learning}},
    archiveprefix = {arXiv},
    eprint = {2409.16495},
    primaryclass = {cs.LG},
    url = {https://arxiv.org/abs/2409.16495},
    year = {2024}
}

@inproceedings{hudson2023trillion,
    author = {Hudson, Nathaniel C and Pauloski, J. Gregory and Baughman, Matt and Kamatar, Alok and Sakarvadia, Mansi and Ward, Logan and Chard, Ryan and Bauer, Andr\'{e} and Levental, Maksim and Wang, Wenyi and Engler, Will and Price Skelly, Owen and Blaiszik, Ben and Stevens, Rick and Chard, Kyle and Foster, Ian},
    title = {{Trillion Parameter AI Serving Infrastructure for Scientific Discovery: A Survey and Vision}},
    abstract = {Deep learning methods are transforming research, enabling new techniques, and ultimately leading to new discoveries. As the demand for more capable AI models continues to grow, we are now entering an era of Trillion Parameter Models (TPM), or models with more than a trillion parameters---such as Huawei's PanGu-Σ. We describe a vision for the ecosystem of TPM users and providers that caters to the specific needs of the scientific community. We then outline the significant technical challenges and open problems in system design for serving TPMs to enable scientific research and discovery. Specifically, we describe the requirements of a comprehensive software stack and interfaces to support the diverse and flexible requirements of researchers.},
    address = {New York, NY, USA},
    articleno = {15},
    booktitle = {Proceedings of the IEEE/ACM 10th International Conference on Big Data Computing, Applications and Technologies},
    doi = {10.1145/3632366.3632396},
    isbn = {9798400704734},
    keywords = {artificial intelligence, grid computing, deep learning applications, systems design, survey},
    location = {<conf-loc>, <city>Taormina (Messina)</city>, <country>Italy</country>, </conf-loc>},
    numpages = {10},
    publisher = {Association for Computing Machinery},
    series = {BDCAT '23},
    url = {https://doi.org/10.1145/3632366.3632396},
    year = {2024}
}

@article{pauloski2022kfac,
    author = {Pauloski, J. Gregory and Huang, Lei and Xu, Weijia and Chard, Kyle and Foster, Ian T. and Zhang, Zhao},
    title = {{Deep Neural Network Training With Distributed K-FAC}},
    doi = {10.1109/TPDS.2022.3161187},
    journal = {IEEE Transactions on Parallel and Distributed Systems},
    number = {12},
    pages = {3616-3627},
    volume = {33},
    year = {2022}
}

@inproceedings{pauloski2021kaisa,
    author = {Pauloski, J. Gregory and Huang, Qi and Huang, Lei and Venkataraman, Shivaram and Chard, Kyle and Foster, Ian and Zhang, Zhao},
    title = {{KAISA: An Adaptive Second-Order Optimizer Framework for Deep Neural Networks}},
    abstract = {Kronecker-factored Approximate Curvature (K-FAC) has recently been shown to converge faster in deep neural network (DNN) training than stochastic gradient descent (SGD); however, K-FAC's larger memory footprint hinders its applicability to large models. We present KAISA, a K-FAC-enabled, Adaptable, Improved, and ScAlable second-order optimizer framework that adapts the memory footprint, communication, and computation given specific models and hardware to improve performance and increase scalability. We quantify the tradeoffs between memory and communication cost and evaluate KAISA on large models, including ResNet-50, Mask R-CNN, U-Net, and BERT, on up to 128 NVIDIA A100 GPUs. Compared to the original optimizers, KAISA converges 18.1--36.3% faster across applications with the same global batch size. Under a fixed memory budget, KAISA converges 32.5% and 41.6% faster in ResNet-50 and BERT-Large, respectively. KAISA can balance memory and communication to achieve scaling efficiency equal to or better than the baseline optimizers.},
    address = {New York, NY, USA},
    articleno = {13},
    booktitle = {Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis},
    doi = {10.1145/3458817.3476152},
    isbn = {9781450384421},
    keywords = {second-order optimization, machine learning, distributed computing, K-FAC, data-parallel algorithms},
    location = {St. Louis, Missouri},
    numpages = {14},
    publisher = {Association for Computing Machinery},
    series = {SC '21},
    url = {https://doi.org/10.1145/3458817.3476152},
    year = {2021}
}

@inproceedings{pauloski2020kfac,
    author = {Pauloski, J. Gregory and Zhang, Zhao and Huang, Lei and Xu, Weijia and Foster, Ian T.},
    title = {{Convolutional Neural Network Training with Distributed K-FAC}},
    abstract = {Training neural networks with many processors can reduce time-to-solution; however, it is challenging to maintain convergence and efficiency at large scales. The Kronecker-factored Approximate Curvature (K-FAC) was recently proposed as an approximation of the Fisher Information Matrix that can be used in natural gradient optimizers. We investigate here a scalable K-FAC design and its applicability in convolutional neural network (CNN) training at scale. We study optimization techniques such as layer-wise distribution strategies, inverse-free second-order gradient evaluation, and dynamic K-FAC update decoupling to reduce training time while preserving convergence. We use residual neural networks (ResNet) applied to the CIFAR-10 and ImageNet-1k datasets to evaluate the correctness and scalability of our K-FAC gradient preconditioner. With ResNet-50 on the ImageNet-1k dataset, our distributed K-FAC implementation converges to the 75.9% MLPerf baseline in 18--25% less time than does the classic stochastic gradient descent (SGD) optimizer across scales on a GPU cluster.},
    articleno = {94},
    booktitle = {Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis},
    doi = {10.5555/3433701.3433826},
    isbn = {9781728199986},
    keywords = {optimization methods, neural networks, high performance computing, scalability},
    location = {Atlanta, Georgia},
    numpages = {14},
    publisher = {IEEE Press},
    series = {SC '20},
    year = {2020}
}

@inproceedings{zhang2020compressed,
    author = {Z. {Zhang} and L. {Huang} and J. G. {Pauloski} and I. T. {Foster}},
    title = {{Efficient I/O for Neural Network Training with Compressed Data}},
    booktitle = {2020 IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
    doi = {10.1109/IPDPS47924.2020.00050},
    number = {},
    pages = {409-418},
    volume = {},
    year = {2020}
}

@inproceedings{zhang2019aggregating,
    author = {Z. {Zhang} and L. {Huang} and J. G. {Pauloski} and I. {Foster}},
    title = {{Aggregating Local Storage for Scalable Deep Learning I/O}},
    booktitle = {2019 IEEE/ACM Third Workshop on Deep Learning on Supercomputers (DLS)},
    doi = {10.1109/DLS49591.2019.00014},
    number = {},
    pages = {69-75},
    volume = {},
    year = {2019}
}

@article{ward2024colmena,
    author = {Logan Ward and J. Gregory Pauloski and Valerie Hayot-Sasson and Yadu Babuji and Alexander Brace and Ryan Chard and Kyle Chard and Rajeev Thakur and Ian Foster},
    title = {{Employing Artificial Intelligence to Steer Exascale Workflows with Colmena}},
    doi = {10.1177/10943420241288242},
    eprint = {https://doi.org/10.1177/10943420241288242},
    journal = {The International Journal of High Performance Computing Applications},
    number = {0},
    pages = {10943420241288242},
    url = {https://doi.org/10.1177/10943420241288242},
    volume = {0},
    year = {0}
}

@misc{song2023deepspeed4science,
    author = {Shuaiwen Leon Song and Bonnie Kruft and Minjia Zhang and Conglong Li and Shiyang Chen and Chengming Zhang and Masahiro Tanaka and Xiaoxia Wu and Jeff Rasley and Ammar Ahmad Awan and Connor Holmes and Martin Cai and Adam Ghanem and Zhongzhu Zhou and Yuxiong He and Pete Luferenko and Divya Kumar and Jonathan Weyn and Ruixiong Zhang and Sylwester Klocek and Volodymyr Vragov and Mohammed AlQuraishi and Gustaf Ahdritz and Christina Floristean and Cristina Negri and Rao Kotamarthi and Venkatram Vishwanath and Arvind Ramanathan and Sam Foreman and Kyle Hippe and Troy Arcomano and Romit Maulik and Maxim Zvyagin and Alexander Brace and Bin Zhang and Cindy Orozco Bohorquez and Austin Clyde and Bharat Kale and Danilo Perez-Rivera and Heng Ma and Carla M. Mann and Michael Irvin and J. Gregory Pauloski and Logan Ward and Valerie Hayot and Murali Emani and Zhen Xie and Diangen Lin and Maulik Shukla and Ian Foster and James J. Davis and Michael E. Papka and Thomas Brettin and Prasanna Balaprakash and Gina Tourassi and John Gounley and Heidi Hanson and Thomas E Potok and Massimiliano Lupo Pasini and Kate Evans and Dan Lu and Dalton Lunga and Junqi Yin and Sajal Dash and Feiyi Wang and Mallikarjun Shankar and Isaac Lyngaas and Xiao Wang and Guojing Cong and Pei Zhang and Ming Fan and Siyan Liu and Adolfy Hoisie and Shinjae Yoo and Yihui Ren and William Tang and Kyle Felker and Alexey Svyatkovskiy and Hang Liu and Ashwin Aji and Angela Dalton and Michael Schulte and Karl Schulz and Yuntian Deng and Weili Nie and Josh Romero and Christian Dallago and Arash Vahdat and Chaowei Xiao and Thomas Gibbs and Anima Anandkumar and Rick Stevens},
    title = {{DeepSpeed4Science Initiative: Enabling Large-Scale Scientific Discovery through Sophisticated AI System Technologies}},
    archiveprefix = {arXiv},
    eprint = {2310.04610},
    primaryclass = {cs.AI},
    year = {2023}
}

@inproceedings{hong2023scholarbert,
    author = {Hong, Zhi and Ajith, Aswathy and Pauloski, J. Gregory and Duede, Eamon and Chard, Kyle and Foster, Ian},
    title = {{The Diminishing Returns of Masked Language Models to Science}},
    abstract = {Transformer-based masked language models such as BERT, trained on general corpora, have shown impressive performance on downstream tasks. It has also been demonstrated that the downstream task performance of such models can be improved by pretraining larger models for longer on more data. In this work, we empirically evaluate the extent to which these results extend to tasks in science. We use 14 domain-specific transformer-based models (including ScholarBERT, a new 770Mparameter science-focused masked language model pretrained on up to 225B tokens) to evaluate the impact of training data, model size, pretraining and finetuning time on 12 downstream scientific tasks. Interestingly, we find that increasing model size, training data, or compute time does not always lead to significant improvements (i.e., {\textgreater}1{\%} F1), if any, in scientific information extraction tasks. We offer possible explanations for this surprising result.},
    address = {Toronto, Canada},
    booktitle = {Findings of the Association for Computational Linguistics: ACL 2023},
    doi = {10.18653/v1/2023.findings-acl.82},
    editor = {Rogers, Anna  and
Boyd-Graber, Jordan  and
Okazaki, Naoaki},
    month = {July},
    pages = {1270--1283},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/2023.findings-acl.82},
    year = {2023}
}

@misc{ward2023colmena,
    author = {Ward, Logan and Pauloski, J. Gregory and Hayot-Sasson, Valerie and Chard, Ryan and Babuji, Yadu and Sivaraman, Ganesh and Choudhury, Sutanay and Chard, Kyle and Thakur, Rajeev and Foster, Ian},
    title = {{Cloud Services Enable Efficient AI-Guided Simulation Workflows across Heterogeneous Resources}},
    copyright = {arXiv.org perpetual, non-exclusive license},
    doi = {10.48550/ARXIV.2303.08803},
    keywords = {Distributed, Parallel, and Cluster Computing (cs.DC), Artificial Intelligence (cs.AI), FOS: Computer and information sciences, FOS: Computer and information sciences},
    publisher = {arXiv},
    url = {https://arxiv.org/abs/2303.08803},
    year = {2023}
}

@article{zvyagin2022genslm,
    author = {Zvyagin, Maxim and Brace, Alexander and Hippe, Kyle and Deng, Yuntian and Zhang, Bin and Orozco Bohorquez, Cindy and Clyde, Austin and Kale, Bharat and Perez-Rivera, Danilo and Ma, Heng and Mann, Carla M. and Irvin, Michael and Pauloski, J. Gregory and Ward, Logan and Hayot, Valerie and Emani, Murali and Foreman, Sam and Xie, Zhen and Lin, Diangen and Shukla, Maulik and Nie, Weili and Romero, Josh and Dallago, Christian and Vahdat, Arash and Xiao, Chaowei and Gibbs, Thomas and Foster, Ian and Davis, James J. and Papka, Michael E. and Brettin, Thomas and Stevens, Rick and Anandkumar, Anima and Vishwanath, Venkatram and Ramanathan, Arvind},
    title = {{GenSLMs: Genome-scale language models reveal SARS-CoV-2 evolutionary dynamics}},
    abstract = {Our work seeks to transform how new and emergent variants of pandemic causing viruses, specially SARS-CoV-2, are identified and classified. By adapting large language models (LLMs) for genomic data, we build genome-scale language models (GenSLMs) which can learn the evolutionary landscape of SARS-CoV-2 genomes. By pretraining on over 110 million prokaryotic gene sequences, and then finetuning a SARS-CoV-2 specific model on 1.5 million genomes, we show that GenSLM can accurately and rapidly identify variants of concern. Thus, to our knowledge, GenSLM represents one of the first whole genome scale foundation models which can generalize to other prediction tasks. We demonstrate the scaling of GenSLMs on both GPU-based supercomputers and AI-hardware accelerators, achieving over 1.54 zettaflops in training runs. We present initial scientific insights gleaned from examining GenSLMs in tracking the evolutionary dynamics of SARS-CoV-2, noting that its full potential on large biological data is yet to be realized.Competing Interest StatementThe authors have declared no competing interest.},
    doi = {10.1101/2022.10.10.511571},
    elocation-id = {2022.10.10.511571},
    eprint = {https://www.biorxiv.org/content/early/2022/10/11/2022.10.10.511571.full.pdf},
    journal = {bioRxiv},
    publisher = {Cold Spring Harbor Laboratory},
    url = {https://www.biorxiv.org/content/early/2022/10/11/2022.10.10.511571},
    year = {2022}
}

@inproceedings{ward2021colmena,
    author = {Ward, Logan and Sivaraman, Ganesh and Pauloski, J. Gregory and Babuji, Yadu and Chard, Ryan and Dandu, Naveen and Redfern, Paul C. and Assary, Rajeev S. and Chard, Kyle and Curtiss, Larry A. and Thakur, Rajeev and Foster, Ian},
    title = {{Colmena: Scalable Machine-Learning-Based Steering of Ensemble Simulations for High Performance Computing}},
    booktitle = {2021 IEEE/ACM Workshop on Machine Learning in High Performance Computing Environments (MLHPC)},
    doi = {10.1109/MLHPC54614.2021.00007},
    number = {},
    pages = {9-20},
    volume = {},
    year = {2021}
}

@article{hong2021moleculesnlp,
    author = {Hong, Zhi and Pauloski, J. Gregory and Ward, Logan and Chard, Kyle and Blaiszik, Ben and Foster, Ian},
    title = {{Models and Processes to Extract Drug-like Molecules From Natural Language Text}},
    abstract = {Researchers worldwide are seeking to repurpose existing drugs or discover new drugs to counter the disease caused by severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2). A promising source of candidates for such studies is molecules that have been reported in the scientific literature to be drug-like in the context of viral research. However, this literature is too large for human review and features unusual vocabularies for which existing named entity recognition (NER) models are ineffective. We report here on a project that leverages both human and artificial intelligence to detect references to such molecules in free text. We present 1) a iterative model-in-the-loop method that makes judicious use of scarce human expertise in generating training data for a NER model, and 2) the application and evaluation of this method to the problem of identifying drug-like molecules in the COVID-19 Open Research Dataset Challenge (CORD-19) corpus of 198,875 papers. We show that by repeatedly presenting human labelers only with samples for which an evolving NER model is uncertain, our human-machine hybrid pipeline requires only modest amounts of non-expert human labeling time (tens of hours to label 1778 samples) to generate an NER model with an F-1 score of 80.5%—on par with that of non-expert humans—and when applied to CORD’19, identifies 10,912 putative drug-like molecules. This enriched the computational screening team’s targets by 3,591 molecules, of which 18 ranked in the top 0.1% of all 6.6 million molecules screened for docking against the 3CLPro protein.},
    doi = {10.3389/fmolb.2021.636077},
    issn = {2296-889X},
    journal = {Frontiers in Molecular Biosciences},
    pages = {826},
    url = {https://www.frontiersin.org/article/10.3389/fmolb.2021.636077},
    volume = {8},
    year = {2021}
}

@inproceedings{gates2019glioma,
    author = {Gates, Evan and Pauloski, J. Gregory and Schellingerhout, Dawid and Fuentes, David},
    title = {{Glioma Segmentation and a Simple Accurate Model for Overall Survival Prediction}},
    abstract = {Brain tumor segmentation is a challenging task necessary for quantitative tumor analysis and diagnosis. We apply a multi-scale convolutional neural network based on the DeepMedic to segment glioma subvolumes provided in the 2018 MICCAI Brain Tumor Segmentation Challenge. We go on to extract intensity and shape features from the images and cross-validate machine learning models to predict overall survival. Using only the mean FLAIR intensity, nonenhancing tumor volume, and patient age we are able to predict patient overall survival with reasonable accuracy.},
    address = {Cham},
    booktitle = {Brainlesion: Glioma, Multiple Sclerosis, Stroke and Traumatic Brain Injuries},
    editor = {Crimi, Alessandro and Bakas, Spyridon and Kuijf, Hugo and Keyvan, Farahani and Reyes, Mauricio and van Walsum, Theo},
    isbn = {978-3-030-11726-9},
    pages = {476--484},
    publisher = {Springer International Publishing},
    year = {2019}
}