merge origin

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -9,4 +9,29 @@ app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# starline
+**St**rict coloring m**a**chine fo**r** **line** drawings.
+
+
+![image](https://github.com/mattyamonaca/starline/assets/48423148/eae07a6e-9c7b-4292-8c70-dac8ec8eeb7b)
+
+
+https://github.com/mattyamonaca/starline/assets/48423148/8199c65c-a19f-42e9-aab7-df5ed6ef5b4c
+
+
+# Usage
+- ```python app.py```
+- Input the line drawing you wish to color (The background should be transparent).
+- Input a prompt describing the color you want to add.
+
+- 背景を透過した状態で線画を入力します
+- 付けたい色を説明するプロンプトを入力します
+
+# Precautions
+- Image size 1024 x 1024 is recommended.
+- Aliasing is a beta version.
+- Areas finely surrounded by line drawings cannot be colored.
+
+- 画像サイズは1024×1024を推奨します
+- エイリアス処理はβ版です。より線画に忠実であることを求める場合は2値線画を推奨します
+- 線画で細かく囲まれた部分は着色できません。着色できない部分は透過した状態で出力されます。

app.py

@@ -0,0 +1,92 @@
+import gradio as gr
+import sys
+from starline import process
+
+from utils import load_cn_model, load_cn_config, randomname
+from convertor import pil2cv, cv2pil
+
+from sd_model import get_cn_pipeline, generate, get_cn_detector
+import cv2
+import os
+import numpy as np
+from PIL import Image
+
+
+path = os.getcwd()
+output_dir = f"{path}/output"
+input_dir = f"{path}/input"
+cn_lineart_dir = f"{path}/controlnet/lineart"
+
+load_cn_model(cn_lineart_dir)
+load_cn_config(cn_lineart_dir)
+
+class webui:
+    def __init__(self):
+        self.demo = gr.Blocks()
+
+    def undercoat(self, input_image, pos_prompt, neg_prompt, alpha_th):
+        org_line_image = input_image
+        image = pil2cv(input_image)
+        image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
+
+        index = np.where(image[:, :, 3] == 0)
+        image[index] = [255, 255, 255, 255]
+        input_image = cv2pil(image)
+
+        pipe = get_cn_pipeline()
+        detectors = get_cn_detector(input_image.resize((1024, 1024), Image.ANTIALIAS))
+        
+
+        gen_image = generate(pipe, detectors, pos_prompt, neg_prompt)
+        output = process(gen_image.resize((image.shape[1], image.shape[0]), Image.ANTIALIAS) , org_line_image, alpha_th)
+
+        output = output.resize((image.shape[1], image.shape[0]) , Image.ANTIALIAS)
+
+
+        output = Image.alpha_composite(output, org_line_image)
+        name = randomname(10)
+        output.save(f"{output_dir}/output_{name}.png")
+        #output = pil2cv(output)
+        file_name = f"{output_dir}/output_{name}.png"
+
+        return output, file_name
+
+
+
+    def launch(self, share):
+        with self.demo:
+            with gr.Row():
+                with gr.Column():
+                    input_image = gr.Image(type="pil", image_mode="RGBA")
+
+                    pos_prompt = gr.Textbox(max_lines=1000, label="positive prompt")                    
+                    neg_prompt = gr.Textbox(max_lines=1000, label="negative prompt")
+
+                    alpha_th = gr.Slider(maximum = 255, value=100, label = "alpha threshold")
+
+                    submit = gr.Button(value="Start")
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Tab("output"):
+                            output_0 = gr.Image()
+
+                    output_file = gr.File()
+            submit.click(
+                self.undercoat, 
+                inputs=[input_image, pos_prompt, neg_prompt, alpha_th], 
+                outputs=[output_0, output_file]
+            )
+
+        self.demo.queue()
+        self.demo.launch(share=share)
+
+
+if __name__ == "__main__":
+    ui = webui()
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "share":
+            ui.launch(share=True)
+        else:
+            ui.launch(share=False)
+    else:
+        ui.launch(share=False)

convertor.py

@@ -0,0 +1,102 @@
+import pandas as pd
+import numpy as np
+from skimage import color 
+from PIL import Image
+
+
+def skimage_rgb2lab(rgb):
+    return color.rgb2lab(rgb.reshape(1,1,3))
+
+
+def rgb2df(img):
+    h, w, _ = img.shape
+    x_l, y_l = np.meshgrid(np.arange(h), np.arange(w), indexing='ij')
+    r, g, b = img[:,:,0], img[:,:,1], img[:,:,2]
+    df = pd.DataFrame({
+        "x_l": x_l.ravel(),
+        "y_l": y_l.ravel(),
+        "r": r.ravel(),
+        "g": g.ravel(),
+        "b": b.ravel(),
+    })
+    return df
+
+def mask2df(mask):
+    h, w = mask.shape
+    x_l, y_l = np.meshgrid(np.arange(h), np.arange(w), indexing='ij')
+    flg = mask.astype(int)
+    df = pd.DataFrame({
+        "x_l_m": x_l.ravel(),
+        "y_l_m": y_l.ravel(),
+        "m_flg": flg.ravel(),
+    })
+    return df
+
+
+def rgba2df(img):
+    h, w, _ = img.shape
+    x_l, y_l = np.meshgrid(np.arange(h), np.arange(w), indexing='ij')
+    r, g, b, a = img[:,:,0], img[:,:,1], img[:,:,2], img[:,:,3]
+    df = pd.DataFrame({
+        "x_l": x_l.ravel(),
+        "y_l": y_l.ravel(),
+        "r": r.ravel(),
+        "g": g.ravel(),
+        "b": b.ravel(),
+        "a": a.ravel()
+    })
+    return df
+
+def hsv2df(img):
+    x_l, y_l = np.meshgrid(np.arange(img.shape[0]), np.arange(img.shape[1]), indexing='ij')
+    h, s, v = np.transpose(img, (2, 0, 1))
+    df = pd.DataFrame({'x_l': x_l.flatten(), 'y_l': y_l.flatten(), 'h': h.flatten(), 's': s.flatten(), 'v': v.flatten()})
+    return df
+
+def df2rgba(img_df):
+    r_img = img_df.pivot_table(index="x_l", columns="y_l",values= "r").reset_index(drop=True).values
+    g_img = img_df.pivot_table(index="x_l", columns="y_l",values= "g").reset_index(drop=True).values
+    b_img = img_df.pivot_table(index="x_l", columns="y_l",values= "b").reset_index(drop=True).values
+    a_img = img_df.pivot_table(index="x_l", columns="y_l",values= "a").reset_index(drop=True).values
+    df_img = np.stack([r_img, g_img, b_img, a_img], 2).astype(np.uint8)
+    return df_img
+
+def df2bgra(img_df):
+    r_img = img_df.pivot_table(index="x_l", columns="y_l",values= "r").reset_index(drop=True).values
+    g_img = img_df.pivot_table(index="x_l", columns="y_l",values= "g").reset_index(drop=True).values
+    b_img = img_df.pivot_table(index="x_l", columns="y_l",values= "b").reset_index(drop=True).values
+    a_img = img_df.pivot_table(index="x_l", columns="y_l",values= "a").reset_index(drop=True).values
+    df_img = np.stack([b_img, g_img, r_img, a_img], 2).astype(np.uint8)
+    return df_img
+
+def df2rgb(img_df):
+    r_img = img_df.pivot_table(index="x_l", columns="y_l",values= "r").reset_index(drop=True).values
+    g_img = img_df.pivot_table(index="x_l", columns="y_l",values= "g").reset_index(drop=True).values
+    b_img = img_df.pivot_table(index="x_l", columns="y_l",values= "b").reset_index(drop=True).values
+    df_img = np.stack([r_img, g_img, b_img], 2).astype(np.uint8)
+    return df_img
+
+def pil2cv(image):
+    new_image = np.array(image, dtype=np.uint8)
+    if new_image.ndim == 2:
+        pass
+    elif new_image.shape[2] == 3:
+        new_image = new_image[:, :, ::-1]
+    elif new_image.shape[2] == 4:
+        new_image = new_image[:, :, [2, 1, 0, 3]]
+    return new_image
+
+def cv2pil(image):
+    new_image = image.copy()
+    if new_image.ndim == 2:
+        pass
+    elif new_image.shape[2] == 3:
+        new_image = new_image[:, :, ::-1]
+    elif new_image.shape[2] == 4:
+        new_image = new_image[:, :, [2, 1, 0, 3]]
+    new_image = Image.fromarray(new_image)
+    return new_image
+
+
+
+

requirements.txt

@@ -0,0 +1,11 @@
+opencv-python==4.7.0.68
+pandas==1.5.3
+gradio==3.16.2
+scikit-learn==1.2.1
+scikit-image==0.19.3
+Pillow==9.4.0
+tqdm==4.63.0
+diffusers==0.27.2
+gradio==3.16.2
+gradio_client==0.2.5
+

sd_model.py

@@ -0,0 +1,64 @@
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL
+import torch
+
+device = "cuda"
+
+def get_cn_pipeline():
+    controlnets = [
+        ControlNetModel.from_pretrained("./controlnet/lineart", torch_dtype=torch.float16, use_safetensors=True),
+        ControlNetModel.from_pretrained("mattyamonaca/controlnet_line2line_xl", torch_dtype=torch.float16)
+    ]
+
+    vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
+    pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+        "cagliostrolab/animagine-xl-3.1", controlnet=controlnets, vae=vae, torch_dtype=torch.float16
+    )
+
+    pipe.enable_model_cpu_offload()
+
+    #if pipe.safety_checker is not None:
+    #    pipe.safety_checker = lambda images, **kwargs: (images, [False])
+    
+    #pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    #pipe.to(device)
+
+    return pipe
+
+def invert_image(img):
+    # 画像を読み込む
+    # 画像をグレースケールに変換（もしもともと白黒でない場合）
+    img = img.convert('L')
+    # 画像の各ピクセルを反転
+    inverted_img = img.point(lambda p: 255 - p)
+    # 反転した画像を保存
+    return inverted_img
+
+
+def get_cn_detector(image):
+    #lineart_anime = LineartAnimeDetector.from_pretrained("lllyasviel/Annotators")
+    #canny = CannyDetector()
+    #lineart_anime_img = lineart_anime(image)
+    #canny_img = canny(image)
+    #canny_img = canny_img.resize((lineart_anime(image).width, lineart_anime(image).height))
+    re_image = invert_image(image)
+    
+    
+    detectors = [re_image, image]
+    print(detectors)
+    return detectors
+
+def generate(pipe, detectors, prompt, negative_prompt):
+    default_pos = "1girl, bestquality, 4K, ((white background)), no background"
+    default_neg = "shadow, (worst quality, low quality:1.2), (lowres:1.2), (bad anatomy:1.2), (greyscale, monochrome:1.4)"
+    prompt = default_pos + prompt 
+    negative_prompt = default_neg + negative_prompt 
+    print(type(pipe))
+    image = pipe(
+                prompt=prompt,
+                negative_prompt = negative_prompt,
+                image=detectors,
+                num_inference_steps=50,
+                controlnet_conditioning_scale=[1.0, 0.2],
+            ).images[0]
+    return image

starline.py

@@ -0,0 +1,268 @@
+from PIL import Image, ImageFilter
+from collections import defaultdict
+from skimage import color as sk_color
+from PIL import Image
+from tqdm import tqdm
+from skimage.color import deltaE_ciede2000, rgb2lab
+import cv2
+import numpy as np
+
+
+def replace_color(image, color_1, color_2, alpha_np):
+    # 画像データを配列に変換
+    data = np.array(image)
+
+    # RGBAモードの画像であるため、形状変更時に4チャネルを考慮
+    original_shape = data.shape
+    data = data.reshape(-1, 4)  # RGBAのため、4チャネルでフラット化
+
+    # color_1のマッチングを検索する際にはRGB値のみを比較
+    matches = np.all(data[:, :3] == color_1, axis=1)
+
+    # 変更を追跡するためのフラグ
+    nochange_count = 0
+    idx = 0
+
+    while np.any(matches):
+        idx += 1
+        new_matches = np.zeros_like(matches)
+        match_num = np.sum(matches)
+        for i in tqdm(range(len(data))):
+            if matches[i]:
+                x, y = divmod(i, original_shape[1])
+                neighbors = [
+                    (x-1, y), (x+1, y), (x, y-1), (x, y+1)  # 上下左右
+                ]
+                replacement_found = False
+                for nx, ny in neighbors:
+                    if 0 <= nx < original_shape[0] and 0 <= ny < original_shape[1]:
+                        ni = nx * original_shape[1] + ny
+                        # RGBのみ比較し、アルファは無視
+                        if not np.all(data[ni, :3] == color_1, axis=0) and not np.all(data[ni, :3] == color_2, axis=0):
+                            data[i, :3] = data[ni, :3]  # RGB値のみ更新
+                            replacement_found = True
+                            continue
+                if not replacement_found:
+                    new_matches[i] = True
+        matches = new_matches
+        if match_num == np.sum(matches):
+             nochange_count += 1
+        if nochange_count > 5:
+            break
+
+    # 最終的な画像をPIL形式で返す
+    data = data.reshape(original_shape)
+    data[:, :, 3] = 255 - alpha_np
+    return Image.fromarray(data, 'RGBA')
+
+def recolor_lineart_and_composite(lineart_image, base_image, new_color, alpha_th):
+    """
+    Recolor an RGBA lineart image to a single new color while preserving alpha, and composite it over a base image.
+
+    Args:
+    lineart_image (PIL.Image): The lineart image with RGBA channels.
+    base_image (PIL.Image): The base image to composite onto.
+    new_color (tuple): The new RGB color for the lineart (e.g., (255, 0, 0) for red).
+
+    Returns:
+    PIL.Image: The composited image with the recolored lineart on top.
+    """
+    # Ensure images are in RGBA mode
+    if lineart_image.mode != 'RGBA':
+        lineart_image = lineart_image.convert('RGBA')
+    if base_image.mode != 'RGBA':
+        base_image = base_image.convert('RGBA')
+
+    # Extract the alpha channel from the lineart image
+    r, g, b, alpha = lineart_image.split()
+
+    alpha_np = np.array(alpha)
+    alpha_np[alpha_np < alpha_th] = 0
+    alpha_np[alpha_np >= alpha_th] = 255
+    
+    new_alpha = Image.fromarray(alpha_np)
+
+    # Create a new image using the new color and the alpha channel from the original lineart
+    new_lineart_image = Image.merge('RGBA', (
+        Image.new('L', lineart_image.size, int(new_color[0])),
+        Image.new('L', lineart_image.size, int(new_color[1])),
+        Image.new('L', lineart_image.size, int(new_color[2])),
+        new_alpha
+    ))
+
+    # Composite the new lineart image over the base image
+    composite_image = Image.alpha_composite(base_image, new_lineart_image)
+
+    return composite_image, alpha_np
+
+
+def thicken_and_recolor_lines(base_image, lineart, thickness=3, new_color=(0, 0, 0)):
+    """
+    Thicken the lines of a lineart image, recolor them, and composite onto another image,
+    while preserving the transparency of the original lineart.
+
+    Args:
+    base_image (PIL.Image): The base image to composite onto.
+    lineart (PIL.Image): The lineart image with transparent background.
+    thickness (int): The desired thickness of the lines.
+    new_color (tuple): The new color to apply to the lines (R, G, B).
+
+    Returns:
+    PIL.Image: The image with the recolored and thickened lineart composited on top.
+    """
+    # Ensure both images are in RGBA format
+    if base_image.mode != 'RGBA':
+        base_image = base_image.convert('RGBA')
+    if lineart.mode != 'RGB':
+        lineart = lineart.convert('RGBA')
+    
+    # Convert the lineart image to OpenCV format
+    lineart_cv = np.array(lineart)
+    
+    white_pixels = np.sum(lineart_cv == 255)
+    black_pixels = np.sum(lineart_cv == 0)
+
+    
+    lineart_gray = cv2.cvtColor(lineart_cv, cv2.COLOR_RGBA2GRAY)
+
+    if white_pixels > black_pixels:
+        lineart_gray = cv2.bitwise_not(lineart_gray)
+    
+    
+    # Thicken the lines using OpenCV
+    kernel = np.ones((thickness, thickness), np.uint8)
+    lineart_thickened = cv2.dilate(lineart_gray, kernel, iterations=1)
+    lineart_thickened = cv2.bitwise_not(lineart_thickened)
+    # Create a new RGBA image for the recolored lineart
+    lineart_recolored = np.zeros_like(lineart_cv)
+    lineart_recolored[:, :, :3] = new_color  # Set new RGB color
+
+    lineart_recolored[:, :, 3] = np.where(lineart_thickened  < 250, 255, 0)  # Blend alpha with thickened lines
+
+    # Convert back to PIL Image
+    lineart_recolored_pil = Image.fromarray(lineart_recolored, 'RGBA')
+    
+    # Composite the thickened and recolored lineart onto the base image
+    combined_image = Image.alpha_composite(base_image, lineart_recolored_pil)
+
+    
+    return combined_image
+
+
+
+def generate_distant_colors(consolidated_colors, distance_threshold):
+    """
+    Generate new RGB colors that are at least 'distance_threshold' CIEDE2000 units away from given colors.
+
+    Args:
+    consolidated_colors (list of tuples): List of ((R, G, B), count) tuples.
+    distance_threshold (float): The minimum CIEDE2000 distance from the given colors.
+
+    Returns:
+    list of tuples: List of new RGB colors that meet the distance requirement.
+    """
+    #new_colors = []
+    # Convert the consolidated colors to LAB
+    consolidated_lab = [rgb2lab(np.array([color], dtype=np.float32) / 255.0).reshape(3) for color, _ in consolidated_colors]
+
+    # Try to find a distant color
+    max_attempts = 10000
+    for _ in range(max_attempts):
+        # Generate a random color in RGB and convert to LAB
+        random_rgb = np.random.randint(0, 256, size=3)
+        random_lab = rgb2lab(np.array([random_rgb], dtype=np.float32) / 255.0).reshape(3)
+        for base_color_lab in consolidated_lab:
+            # Calculate the CIEDE2000 distance
+            distance = deltaE_ciede2000(base_color_lab, random_lab)
+            if distance <= distance_threshold:
+                break
+        new_color = tuple(random_rgb)
+        break
+    return new_color
+                
+
+
+def consolidate_colors(major_colors, threshold):
+    """
+    Consolidate similar colors in the major_colors list based on the CIEDE2000 metric.
+
+    Args:
+    major_colors (list of tuples): List of ((R, G, B), count) tuples.
+    threshold (float): Threshold for CIEDE2000 color difference.
+
+    Returns:
+    list of tuples: Consolidated list of ((R, G, B), count) tuples.
+    """
+    # Convert RGB to LAB
+    colors_lab = [rgb2lab(np.array([[color]], dtype=np.float32)/255.0).reshape(3) for color, _ in major_colors]
+    n = len(colors_lab)
+
+    # Find similar colors and consolidate
+    i = 0
+    while i < n:
+        j = i + 1
+        while j < n:
+            delta_e = deltaE_ciede2000(colors_lab[i], colors_lab[j])
+            if delta_e < threshold:
+                # Compare counts and consolidate to the color with the higher count
+                if major_colors[i][1] >= major_colors[j][1]:
+                    major_colors[i] = (major_colors[i][0], major_colors[i][1] + major_colors[j][1])
+                    major_colors.pop(j)
+                    colors_lab.pop(j)
+                else:
+                    major_colors[j] = (major_colors[j][0], major_colors[j][1] + major_colors[i][1])
+                    major_colors.pop(i)
+                    colors_lab.pop(i)
+                n -= 1
+                continue
+            j += 1
+        i += 1
+
+    return major_colors
+
+
+
+
+def get_major_colors(image, threshold_percentage=0.01):
+    """
+    Analyze an image to find the major RGB values based on a threshold percentage.
+
+    Args:
+    image (PIL.Image): The image to analyze.
+    threshold_percentage (float): The percentage threshold to consider a color as major.
+
+    Returns:
+    list of tuples: A list of (color, count) tuples for colors that are more frequent than the threshold.
+    """
+    # Convert image to RGB if it's not
+    if image.mode != 'RGB':
+        image = image.convert('RGB')
+
+    # Count each color
+    color_count = defaultdict(int)
+    for pixel in image.getdata():
+        color_count[pixel] += 1
+
+    # Total number of pixels
+    total_pixels = image.width * image.height
+
+    # Filter colors to find those above the threshold
+    major_colors = [(color, count) for color, count in color_count.items()
+                    if (count / total_pixels) >= threshold_percentage]
+
+    return major_colors
+
+
+def process(image, lineart, alpha_th):
+    org = image
+    
+    major_colors = get_major_colors(image, threshold_percentage=0.05)
+    major_colors = consolidate_colors(major_colors, 10)
+    new_color_1 = generate_distant_colors(major_colors, 100)
+    image = thicken_and_recolor_lines(org, lineart, thickness=5, new_color=new_color_1)
+    major_colors.append((new_color_1, 0))
+    new_color_2 = generate_distant_colors(major_colors, 100)
+    image, alpha_np = recolor_lineart_and_composite(lineart, image, new_color_2, alpha_th)
+    image = replace_color(image, new_color_1, new_color_2, alpha_np)
+
+    return image

utils.py

@@ -0,0 +1,53 @@
+import random
+import string
+import os
+
+import requests
+from tqdm import tqdm
+
+
+def randomname(n):
+   randlst = [random.choice(string.ascii_letters + string.digits) for i in range(n)]
+   return ''.join(randlst)
+
+def load_cn_model(model_dir):
+  folder = model_dir
+  file_name = 'diffusion_pytorch_model.safetensors'
+  url = "https://huggingface.co/kataragi/ControlNet-LineartXL/resolve/main/Katarag_lineartXL-fp16.safetensors"
+
+  file_path = os.path.join(folder, file_name)
+  if not os.path.exists(file_path):
+    response = requests.get(url, stream=True)
+
+    total_size = int(response.headers.get('content-length', 0))
+    with open(file_path, 'wb') as f, tqdm(
+            desc=file_name,
+            total=total_size,
+            unit='iB',
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as bar:
+        for data in response.iter_content(chunk_size=1024):
+            size = f.write(data)
+            bar.update(size)
+
+def load_cn_config(model_dir):
+  folder = model_dir
+  file_name = 'config.json'
+  url = "https://huggingface.co/mattyamonaca/controlnet_line2line_xl/resolve/main/config.json"
+
+  file_path = os.path.join(folder, file_name)
+  if not os.path.exists(file_path):
+    response = requests.get(url, stream=True)
+
+    total_size = int(response.headers.get('content-length', 0))
+    with open(file_path, 'wb') as f, tqdm(
+            desc=file_name,
+            total=total_size,
+            unit='iB',
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as bar:
+        for data in response.iter_content(chunk_size=1024):
+            size = f.write(data)
+            bar.update(size)