@article{PanzerBenderGronau2023,
  author    = {Panzer, Marcel and Bender, Benedict and Gronau, Norbert},
  title     = {A deep reinforcement learning based hyper-heuristic for modular production control},
  series = {International journal of production research},
  journal   = {International journal of production research},
  publisher = {Taylor \& Francis},
  address   = {London},
  issn      = {0020-7543},
  doi       = {10.1080/00207543.2023.2233641},
  pages     = {1 -- 22},
  year      = {2023},
  abstract  = {In nowadays production, fluctuations in demand, shortening product life-cycles, and highly configurable products require an adaptive and robust control approach to maintain competitiveness. This approach must not only optimise desired production objectives but also cope with unforeseen machine failures, rush orders, and changes in short-term demand. Previous control approaches were often implemented using a single operations layer and a standalone deep learning approach, which may not adequately address the complex organisational demands of modern manufacturing systems. To address this challenge, we propose a hyper-heuristics control model within a semi-heterarchical production system, in which multiple manufacturing and distribution agents are spread across pre-defined modules. The agents employ a deep reinforcement learning algorithm to learn a policy for selecting low-level heuristics in a situation-specific manner, thereby leveraging system performance and adaptability. We tested our approach in simulation and transferred it to a hybrid production environment. By that, we were able to demonstrate its multi-objective optimisation capabilities compared to conventional approaches in terms of mean throughput time, tardiness, and processing of prioritised orders in a multi-layered production system. The modular design is promising in reducing the overall system complexity and facilitates a quick and seamless integration into other scenarios.},
  language  = {en}
}
@article{PanzerBender2021,
  author    = {Panzer, Marcel and Bender, Benedict},
  title     = {Deep reinforcement learning in production systems},
  series = {International Journal of Production Research},
  volume    = {13},
  journal   = {International Journal of Production Research},
  number    = {60},
  publisher = {Taylor \& Francis},
  address   = {London},
  issn      = {1366-588X},
  doi       = {10.1080/00207543.2021.1973138},
  year      = {2021},
  abstract  = {Shortening product development cycles and fully customizable products pose major challenges for production systems. These not only have to cope with an increased product diversity but also enable high throughputs and provide a high adaptability and robustness to process variations and unforeseen incidents. To overcome these challenges, deep Reinforcement Learning (RL) has been increasingly applied for the optimization of production systems. Unlike other machine learning methods, deep RL operates on recently collected sensor-data in direct interaction with its environment and enables real-time responses to system changes. Although deep RL is already being deployed in production systems, a systematic review of the results has not yet been established. The main contribution of this paper is to provide researchers and practitioners an overview of applications and to motivate further implementations and research of deep RL supported production systems. Findings reveal that deep RL is applied in a variety of production domains, contributing to data-driven and flexible processes. In most applications, conventional methods were outperformed and implementation efforts or dependence on human experience were reduced. Nevertheless, future research must focus more on transferring the findings to real-world systems to analyze safety aspects and demonstrate reliability under prevailing conditions.},
  language  = {en}
}
@article{PanzerGronau2024,
  author    = {Panzer, Marcel and Gronau, Norbert},
  title     = {Enhancing economic efficiency in modular production systems through deep reinforcement learning},
  series = {Procedia CIRP},
  volume    = {121},
  journal   = {Procedia CIRP},
  publisher = {Elsevier},
  address   = {Amsterdam},
  issn      = {2212-8271},
  doi       = {10.1016/j.procir.2023.09.229},
  pages     = {55 -- 60},
  year      = {2024},
  abstract  = {In times of increasingly complex production processes and volatile customer demands, the production adaptability is crucial for a company's profitability and competitiveness. The ability to cope with rapidly changing customer requirements and unexpected internal and external events guarantees robust and efficient production processes, requiring a dedicated control concept at the shop floor level. Yet in today's practice, conventional control approaches remain in use, which may not keep up with the dynamic behaviour due to their scenario-specific and rigid properties. To address this challenge, deep learning methods were increasingly deployed due to their optimization and scalability properties. However, these approaches were often tested in specific operational applications and focused on technical performance indicators such as order tardiness or total throughput. In this paper, we propose a deep reinforcement learning based production control to optimize combined techno-financial performance measures. Based on pre-defined manufacturing modules that are supplied and operated by multiple agents, positive effects were observed in terms of increased revenue and reduced penalties due to lower throughput times and fewer delayed products. The combined modular and multi-staged approach as well as the distributed decision-making further leverage scalability and transferability to other scenarios.},
  language  = {en}
}
@article{PanzerBenderGronau2022,
  author    = {Panzer, Marcel and Bender, Benedict and Gronau, Norbert},
  title     = {Neural agent-based production planning and control},
  series = {Journal of Manufacturing Systems},
  volume    = {65},
  journal   = {Journal of Manufacturing Systems},
  publisher = {Elsevier},
  address   = {Amsterdam},
  issn      = {0278-6125},
  doi       = {10.1016/j.jmsy.2022.10.019},
  pages     = {743 -- 766},
  year      = {2022},
  abstract  = {Nowadays, production planning and control must cope with mass customization, increased fluctuations in demand, and high competition pressures. Despite prevailing market risks, planning accuracy and increased adaptability in the event of disruptions or failures must be ensured, while simultaneously optimizing key process indicators. To manage that complex task, neural networks that can process large quantities of high-dimensional data in real time have been widely adopted in recent years. Although these are already extensively deployed in production systems, a systematic review of applications and implemented agent embeddings and architectures has not yet been conducted. The main contribution of this paper is to provide researchers and practitioners with an overview of applications and applied embeddings and to motivate further research in neural agent-based production. Findings indicate that neural agents are not only deployed in diverse applications, but are also increasingly implemented in multi-agent environments or in combination with conventional methods — leveraging performances compared to benchmarks and reducing dependence on human experience. This not only implies a more sophisticated focus on distributed production resources, but also broadening the perspective from a local to a global scale. Nevertheless, future research must further increase scalability and reproducibility to guarantee a simplified transfer of results to reality.},
  language  = {en}
}