@article{ReimannBuchheimSemmoetal.2022,
  author    = {Reimann, Max and Buchheim, Benito and Semmo, Amir and D{\"o}llner, J{\"u}rgen and Trapp, Matthias},
  title     = {Controlling strokes in fast neural style transfer using content transforms},
  series = {The Visual Computer},
  volume    = {38},
  journal   = {The Visual Computer},
  number    = {12},
  publisher = {Springer},
  address   = {New York},
  issn      = {0178-2789},
  doi       = {10.1007/s00371-022-02518-x},
  pages     = {4019 -- 4033},
  year      = {2022},
  abstract  = {Fast style transfer methods have recently gained popularity in art-related applications as they make a generalized real-time stylization of images practicable. However, they are mostly limited to one-shot stylizations concerning the interactive adjustment of style elements. In particular, the expressive control over stroke sizes or stroke orientations remains an open challenge. To this end, we propose a novel stroke-adjustable fast style transfer network that enables simultaneous control over the stroke size and intensity, and allows a wider range of expressive editing than current approaches by utilizing the scale-variance of convolutional neural networks. Furthermore, we introduce a network-agnostic approach for style-element editing by applying reversible input transformations that can adjust strokes in the stylized output. At this, stroke orientations can be adjusted, and warping-based effects can be applied to stylistic elements, such as swirls or waves. To demonstrate the real-world applicability of our approach, we present StyleTune, a mobile app for interactive editing of neural style transfers at multiple levels of control. Our app allows stroke adjustments on a global and local level. It furthermore implements an on-device patch-based upsampling step that enables users to achieve results with high output fidelity and resolutions of more than 20 megapixels. Our approach allows users to art-direct their creations and achieve results that are not possible with current style transfer applications.},
  language  = {en}
}
@article{ShekharReimannMayeretal.2021,
  author    = {Shekhar, Sumit and Reimann, Max and Mayer, Maximilian and Semmo, Amir and Pasewaldt, Sebastian and D{\"o}llner, J{\"u}rgen and Trapp, Matthias},
  title     = {Interactive photo editing on smartphones via intrinsic decomposition},
  series = {Computer graphics forum : journal of the European Association for Computer Graphics},
  volume    = {40},
  journal   = {Computer graphics forum : journal of the European Association for Computer Graphics},
  publisher = {Blackwell},
  address   = {Oxford},
  issn      = {0167-7055},
  doi       = {10.1111/cgf.142650},
  pages     = {497 -- 510},
  year      = {2021},
  abstract  = {Intrinsic decomposition refers to the problem of estimating scene characteristics, such as albedo and shading, when one view or multiple views of a scene are provided. The inverse problem setting, where multiple unknowns are solved given a single known pixel-value, is highly under-constrained. When provided with correlating image and depth data, intrinsic scene decomposition can be facilitated using depth-based priors, which nowadays is easy to acquire with high-end smartphones by utilizing their depth sensors. In this work, we present a system for intrinsic decomposition of RGB-D images on smartphones and the algorithmic as well as design choices therein. Unlike state-of-the-art methods that assume only diffuse reflectance, we consider both diffuse and specular pixels. For this purpose, we present a novel specularity extraction algorithm based on a multi-scale intensity decomposition and chroma inpainting. At this, the diffuse component is further decomposed into albedo and shading components. We use an inertial proximal algorithm for non-convex optimization (iPiano) to ensure albedo sparsity. Our GPU-based visual processing is implemented on iOS via the Metal API and enables interactive performance on an iPhone 11 Pro. Further, a qualitative evaluation shows that we are able to obtain high-quality outputs. Furthermore, our proposed approach for specularity removal outperforms state-of-the-art approaches for real-world images, while our albedo and shading layer decomposition is faster than the prior work at a comparable output quality. Manifold applications such as recoloring, retexturing, relighting, appearance editing, and stylization are shown, each using the intrinsic layers obtained with our method and/or the corresponding depth data.},
  language  = {en}
}
@article{ReimannKlingbeilPasewaldtetal.2019,
  author    = {Reimann, Max and Klingbeil, Mandy and Pasewaldt, Sebastian and Semmo, Amir and Trapp, Matthias and D{\"o}llner, J{\"u}rgen Roland Friedrich},
  title     = {Locally controllable neural style transfer on mobile devices},
  series = {The Visual Computer},
  volume    = {35},
  journal   = {The Visual Computer},
  number    = {11},
  publisher = {Springer},
  address   = {New York},
  issn      = {0178-2789},
  doi       = {10.1007/s00371-019-01654-1},
  pages     = {1531 -- 1547},
  year      = {2019},
  abstract  = {Mobile expressive rendering gained increasing popularity among users seeking casual creativity by image stylization and supports the development of mobile artists as a new user group. In particular, neural style transfer has advanced as a core technology to emulate characteristics of manifold artistic styles. However, when it comes to creative expression, the technology still faces inherent limitations in providing low-level controls for localized image stylization. In this work, we first propose a problem characterization of interactive style transfer representing a trade-off between visual quality, run-time performance, and user control. We then present MaeSTrO, a mobile app for orchestration of neural style transfer techniques using iterative, multi-style generative and adaptive neural networks that can be locally controlled by on-screen painting metaphors. At this, we enhance state-of-the-art neural style transfer techniques by mask-based loss terms that can be interactively parameterized by a generalized user interface to facilitate a creative and localized editing process. We report on a usability study and an online survey that demonstrate the ability of our app to transfer styles at improved semantic plausibility.},
  language  = {en}
}