Merge pull request #10291 from akx/test-overhaul

Test overhaul

17 files changed, 307 insertions, 326 deletions

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 0708398b..226cf759 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -18,18 +18,53 @@ jobs:
           cache-dependency-path: |
             **/requirements*txt
             launch.py
-      - name: Run tests
-        run: python launch.py --tests test --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
+      - name: Install test dependencies
+        run: pip install wait-for-it -r requirements-test.txt
+        env:
+          PIP_DISABLE_PIP_VERSION_CHECK: "1"
+          PIP_PROGRESS_BAR: "off"
+      - name: Setup environment
+        run: python launch.py --skip-torch-cuda-test --exit
         env:
           PIP_DISABLE_PIP_VERSION_CHECK: "1"
           PIP_PROGRESS_BAR: "off"
           TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu
           WEBUI_LAUNCH_LIVE_OUTPUT: "1"
-      - name: Upload main app stdout-stderr
+          PYTHONUNBUFFERED: "1"
+      - name: Start test server
+        run: >
+          python -m coverage run
+          --data-file=.coverage.server
+          launch.py
+          --skip-prepare-environment
+          --skip-torch-cuda-test
+          --test-server
+          --no-half
+          --disable-opt-split-attention
+          --use-cpu all
+          --add-stop-route
+          2>&1 | tee output.txt &
+      - name: Run tests
+        run: |
+          wait-for-it --service 127.0.0.1:7860 -t 600
+          python -m pytest -vv --junitxml=test/results.xml --cov . --cov-report=xml --verify-base-url test
+      - name: Kill test server
+        if: always()
+        run: curl -vv -XPOST http://127.0.0.1:7860/_stop && sleep 10
+      - name: Show coverage
+        run: |
+          python -m coverage combine .coverage*
+          python -m coverage report -i
+          python -m coverage html -i
+      - name: Upload main app output
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: output
+          path: output.txt
+      - name: Upload coverage HTML
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: stdout-stderr
-          path: |
-            test/stdout.txt
-            test/stderr.txt
+          name: htmlcov
+          path: htmlcov
diff --git a/.gitignore b/.gitignore
index 46654d83..09734267 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,4 +35,5 @@ notification.mp3
 /cache.json*
 /config_states/
 /node_modules
-/package-lock.json
\ No newline at end of file
+/package-lock.json
+/.coverage*
diff --git a/launch.py b/launch.py
index 6e9ca8de..b9b5b709 100644
--- a/launch.py
+++ b/launch.py
@@ -310,12 +310,8 @@ def prepare_environment():
         print("Exiting because of --exit argument")

         exit(0)

 

-    if args.tests and not args.no_tests:

-        exitcode = tests(args.tests)

-        exit(exitcode)

 

-

-def tests(test_dir):

+def configure_for_tests():

     if "--api" not in sys.argv:

         sys.argv.append("--api")

     if "--ckpt" not in sys.argv:

@@ -325,21 +321,8 @@ def tests(test_dir):
         sys.argv.append("--skip-torch-cuda-test")

     if "--disable-nan-check" not in sys.argv:

         sys.argv.append("--disable-nan-check")

-    if "--no-tests" not in sys.argv:

-        sys.argv.append("--no-tests")

-

-    print(f"Launching Web UI in another process for testing with arguments: {' '.join(sys.argv[1:])}")

 

     os.environ['COMMANDLINE_ARGS'] = ""

-    with open(os.path.join(script_path, 'test/stdout.txt'), "w", encoding="utf8") as stdout, open(os.path.join(script_path, 'test/stderr.txt'), "w", encoding="utf8") as stderr:

-        proc = subprocess.Popen([sys.executable, *sys.argv], stdout=stdout, stderr=stderr)

-

-    import test.server_poll

-    exitcode = test.server_poll.run_tests(proc, test_dir)

-

-    print(f"Stopping Web UI process with id {proc.pid}")

-    proc.kill()

-    return exitcode

 

 

 def start():

@@ -351,6 +334,15 @@ def start():
         webui.webui()

 

 

-if __name__ == "__main__":

-    prepare_environment()

+def main():

+    if not args.skip_prepare_environment:

+        prepare_environment()

+

+    if args.test_server:

+        configure_for_tests()

+

     start()

+

+

+if __name__ == "__main__":

+    main()

diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 7bde161e..8625690b 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -12,8 +12,8 @@ parser.add_argument("--skip-torch-cuda-test", action='store_true', help="launch.
 parser.add_argument("--reinstall-xformers", action='store_true', help="launch.py argument: install the appropriate version of xformers even if you have some version already installed")

 parser.add_argument("--reinstall-torch", action='store_true', help="launch.py argument: install the appropriate version of torch even if you have some version already installed")

 parser.add_argument("--update-check", action='store_true', help="launch.py argument: chck for updates at startup")

-parser.add_argument("--tests", type=str, default=None, help="launch.py argument: run tests in the specified directory")

-parser.add_argument("--no-tests", action='store_true', help="launch.py argument: do not run tests even if --tests option is specified")

+parser.add_argument("--test-server", action='store_true', help="launch.py argument: configure server for testing")

+parser.add_argument("--skip-prepare-environment", action='store_true', help="launch.py argument: skip all environment preparation")

 parser.add_argument("--skip-install", action='store_true', help="launch.py argument: skip installation of packages")

 parser.add_argument("--data-dir", type=str, default=os.path.dirname(os.path.dirname(os.path.realpath(__file__))), help="base path where all user data is stored")

 parser.add_argument("--config", type=str, default=sd_default_config, help="path to config which constructs model",)

diff --git a/pyproject.toml b/pyproject.toml
index d4a1bbf4..80541a8f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,3 +30,6 @@ ignore = [
 [tool.ruff.flake8-bugbear]
 # Allow default arguments like, e.g., `data: List[str] = fastapi.Query(None)`.
 extend-immutable-calls = ["fastapi.Depends", "fastapi.security.HTTPBasic"]
+
+[tool.pytest.ini_options]
+base_url = "http://127.0.0.1:7860"
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 00000000..37838ca2
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,3 @@
+pytest-base-url~=2.0
+pytest-cov~=4.0
+pytest~=7.3
diff --git a/test/basic_features/__init__.py b/test/basic_features/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/test/basic_features/__init__.py
+++ /dev/null
diff --git a/test/basic_features/extras_test.py b/test/basic_features/extras_test.py
deleted file mode 100644
index 8ed98747..00000000
--- a/test/basic_features/extras_test.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import os
-import unittest
-import requests
-from gradio.processing_utils import encode_pil_to_base64
-from PIL import Image
-from modules.paths import script_path
-
-class TestExtrasWorking(unittest.TestCase):
-    def setUp(self):
-        self.url_extras_single = "http://localhost:7860/sdapi/v1/extra-single-image"
-        self.extras_single = {
-            "resize_mode": 0,
-            "show_extras_results": True,
-            "gfpgan_visibility": 0,
-            "codeformer_visibility": 0,
-            "codeformer_weight": 0,
-            "upscaling_resize": 2,
-            "upscaling_resize_w": 128,
-            "upscaling_resize_h": 128,
-            "upscaling_crop": True,
-            "upscaler_1": "None",
-            "upscaler_2": "None",
-            "extras_upscaler_2_visibility": 0,
-            "image": encode_pil_to_base64(Image.open(os.path.join(script_path, r"test/test_files/img2img_basic.png")))
-            }
-
-    def test_simple_upscaling_performed(self):
-        self.extras_single["upscaler_1"] = "Lanczos"
-        self.assertEqual(requests.post(self.url_extras_single, json=self.extras_single).status_code, 200)
-
-
-class TestPngInfoWorking(unittest.TestCase):
-    def setUp(self):
-        self.url_png_info = "http://localhost:7860/sdapi/v1/extra-single-image"
-        self.png_info = {
-            "image": encode_pil_to_base64(Image.open(os.path.join(script_path, r"test/test_files/img2img_basic.png")))
-        }
-
-    def test_png_info_performed(self):
-        self.assertEqual(requests.post(self.url_png_info, json=self.png_info).status_code, 200)
-
-
-class TestInterrogateWorking(unittest.TestCase):
-    def setUp(self):
-        self.url_interrogate = "http://localhost:7860/sdapi/v1/extra-single-image"
-        self.interrogate = {
-            "image": encode_pil_to_base64(Image.open(os.path.join(script_path, r"test/test_files/img2img_basic.png"))),
-            "model": "clip"
-        }
-
-    def test_interrogate_performed(self):
-        self.assertEqual(requests.post(self.url_interrogate, json=self.interrogate).status_code, 200)
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/basic_features/img2img_test.py b/test/basic_features/img2img_test.py
deleted file mode 100644
index 5240ec36..00000000
--- a/test/basic_features/img2img_test.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import os
-import unittest
-import requests
-from gradio.processing_utils import encode_pil_to_base64
-from PIL import Image
-from modules.paths import script_path
-
-
-class TestImg2ImgWorking(unittest.TestCase):
-    def setUp(self):
-        self.url_img2img = "http://localhost:7860/sdapi/v1/img2img"
-        self.simple_img2img = {
-            "init_images": [encode_pil_to_base64(Image.open(os.path.join(script_path, r"test/test_files/img2img_basic.png")))],
-            "resize_mode": 0,
-            "denoising_strength": 0.75,
-            "mask": None,
-            "mask_blur": 4,
-            "inpainting_fill": 0,
-            "inpaint_full_res": False,
-            "inpaint_full_res_padding": 0,
-            "inpainting_mask_invert": False,
-            "prompt": "example prompt",
-            "styles": [],
-            "seed": -1,
-            "subseed": -1,
-            "subseed_strength": 0,
-            "seed_resize_from_h": -1,
-            "seed_resize_from_w": -1,
-            "batch_size": 1,
-            "n_iter": 1,
-            "steps": 3,
-            "cfg_scale": 7,
-            "width": 64,
-            "height": 64,
-            "restore_faces": False,
-            "tiling": False,
-            "negative_prompt": "",
-            "eta": 0,
-            "s_churn": 0,
-            "s_tmax": 0,
-            "s_tmin": 0,
-            "s_noise": 1,
-            "override_settings": {},
-            "sampler_index": "Euler a",
-            "include_init_images": False
-            }
-
-    def test_img2img_simple_performed(self):
-        self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
-
-    def test_inpainting_masked_performed(self):
-        self.simple_img2img["mask"] = encode_pil_to_base64(Image.open(os.path.join(script_path, r"test/test_files/img2img_basic.png")))
-        self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
-
-    def test_inpainting_with_inverted_masked_performed(self):
-        self.simple_img2img["mask"] = encode_pil_to_base64(Image.open(os.path.join(script_path, r"test/test_files/img2img_basic.png")))
-        self.simple_img2img["inpainting_mask_invert"] = True
-        self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
-
-    def test_img2img_sd_upscale_performed(self):
-        self.simple_img2img["script_name"] = "sd upscale"
-        self.simple_img2img["script_args"] = ["", 8, "Lanczos", 2.0]
-
-        self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/basic_features/txt2img_test.py b/test/basic_features/txt2img_test.py
deleted file mode 100644
index cb525fbb..00000000
--- a/test/basic_features/txt2img_test.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import unittest
-import requests
-
-
-class TestTxt2ImgWorking(unittest.TestCase):
-    def setUp(self):
-        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
-        self.simple_txt2img = {
-            "enable_hr": False,
-            "denoising_strength": 0,
-            "firstphase_width": 0,
-            "firstphase_height": 0,
-            "prompt": "example prompt",
-            "styles": [],
-            "seed": -1,
-            "subseed": -1,
-            "subseed_strength": 0,
-            "seed_resize_from_h": -1,
-            "seed_resize_from_w": -1,
-            "batch_size": 1,
-            "n_iter": 1,
-            "steps": 3,
-            "cfg_scale": 7,
-            "width": 64,
-            "height": 64,
-            "restore_faces": False,
-            "tiling": False,
-            "negative_prompt": "",
-            "eta": 0,
-            "s_churn": 0,
-            "s_tmax": 0,
-            "s_tmin": 0,
-            "s_noise": 1,
-            "sampler_index": "Euler a"
-        }
-
-    def test_txt2img_simple_performed(self):
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_with_negative_prompt_performed(self):
-        self.simple_txt2img["negative_prompt"] = "example negative prompt"
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_with_complex_prompt_performed(self):
-        self.simple_txt2img["prompt"] = "((emphasis)), (emphasis1:1.1), [to:1], [from::2], [from:to:0.3], [alt|alt1]"
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_not_square_image_performed(self):
-        self.simple_txt2img["height"] = 128
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_with_hrfix_performed(self):
-        self.simple_txt2img["enable_hr"] = True
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_with_tiling_performed(self):
-        self.simple_txt2img["tiling"] = True
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_with_restore_faces_performed(self):
-        self.simple_txt2img["restore_faces"] = True
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_with_vanilla_sampler_performed(self):
-        self.simple_txt2img["sampler_index"] = "PLMS"
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-        self.simple_txt2img["sampler_index"] = "DDIM"
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-        self.simple_txt2img["sampler_index"] = "UniPC"
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_multiple_batches_performed(self):
-        self.simple_txt2img["n_iter"] = 2
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-    def test_txt2img_batch_performed(self):
-        self.simple_txt2img["batch_size"] = 2
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/basic_features/utils_test.py b/test/basic_features/utils_test.py
deleted file mode 100644
index d9e46b5e..00000000
--- a/test/basic_features/utils_test.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import unittest
-import requests
-
-
-class UtilsTests(unittest.TestCase):
-    def setUp(self):
-        self.url_options = "http://localhost:7860/sdapi/v1/options"
-        self.url_cmd_flags = "http://localhost:7860/sdapi/v1/cmd-flags"
-        self.url_samplers = "http://localhost:7860/sdapi/v1/samplers"
-        self.url_upscalers = "http://localhost:7860/sdapi/v1/upscalers"
-        self.url_sd_models = "http://localhost:7860/sdapi/v1/sd-models"
-        self.url_hypernetworks = "http://localhost:7860/sdapi/v1/hypernetworks"
-        self.url_face_restorers = "http://localhost:7860/sdapi/v1/face-restorers"
-        self.url_realesrgan_models = "http://localhost:7860/sdapi/v1/realesrgan-models"
-        self.url_prompt_styles = "http://localhost:7860/sdapi/v1/prompt-styles"
-        self.url_embeddings = "http://localhost:7860/sdapi/v1/embeddings"
-
-    def test_options_get(self):
-        self.assertEqual(requests.get(self.url_options).status_code, 200)
-
-    def test_options_write(self):
-        response = requests.get(self.url_options)
-        self.assertEqual(response.status_code, 200)
-
-        pre_value = response.json()["send_seed"]
-
-        self.assertEqual(requests.post(self.url_options, json={"send_seed": not pre_value}).status_code, 200)
-
-        response = requests.get(self.url_options)
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.json()["send_seed"], not pre_value)
-
-        requests.post(self.url_options, json={"send_seed": pre_value})
-
-    def test_cmd_flags(self):
-        self.assertEqual(requests.get(self.url_cmd_flags).status_code, 200)
-
-    def test_samplers(self):
-        self.assertEqual(requests.get(self.url_samplers).status_code, 200)
-
-    def test_upscalers(self):
-        self.assertEqual(requests.get(self.url_upscalers).status_code, 200)
-
-    def test_sd_models(self):
-        self.assertEqual(requests.get(self.url_sd_models).status_code, 200)
-
-    def test_hypernetworks(self):
-        self.assertEqual(requests.get(self.url_hypernetworks).status_code, 200)
-
-    def test_face_restorers(self):
-        self.assertEqual(requests.get(self.url_face_restorers).status_code, 200)
-
-    def test_realesrgan_models(self):
-        self.assertEqual(requests.get(self.url_realesrgan_models).status_code, 200)
-
-    def test_prompt_styles(self):
-        self.assertEqual(requests.get(self.url_prompt_styles).status_code, 200)
-
-    def test_embeddings(self):
-        self.assertEqual(requests.get(self.url_embeddings).status_code, 200)
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/test/conftest.py b/test/conftest.py
new file mode 100644
index 00000000..0723f62a
--- /dev/null
+++ b/test/conftest.py
@@ -0,0 +1,17 @@
+import os
+
+import pytest
+from PIL import Image
+from gradio.processing_utils import encode_pil_to_base64
+
+test_files_path = os.path.dirname(__file__) + "/test_files"
+
+
+@pytest.fixture(scope="session")  # session so we don't read this over and over
+def img2img_basic_image_base64() -> str:
+    return encode_pil_to_base64(Image.open(os.path.join(test_files_path, "img2img_basic.png")))
+
+
+@pytest.fixture(scope="session")  # session so we don't read this over and over
+def mask_basic_image_base64() -> str:
+    return encode_pil_to_base64(Image.open(os.path.join(test_files_path, "mask_basic.png")))
diff --git a/test/server_poll.py b/test/server_poll.py
deleted file mode 100644
index c732630f..00000000
--- a/test/server_poll.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import unittest
-import requests
-import time
-import os
-from modules.paths import script_path
-
-
-def run_tests(proc, test_dir):
-    timeout_threshold = 240
-    start_time = time.time()
-    while time.time()-start_time < timeout_threshold:
-        try:
-            requests.head("http://localhost:7860/")
-            break
-        except requests.exceptions.ConnectionError:
-            if proc.poll() is not None:
-                break
-    if proc.poll() is None:
-        if test_dir is None:
-            test_dir = os.path.join(script_path, "test")
-        suite = unittest.TestLoader().discover(test_dir, pattern="*_test.py", top_level_dir=test_dir)
-        result = unittest.TextTestRunner(verbosity=2).run(suite)
-        return len(result.failures) + len(result.errors)
-    else:
-        print("Launch unsuccessful")
-        return 1
diff --git a/test/test_extras.py b/test/test_extras.py
new file mode 100644
index 00000000..799d9fad
--- /dev/null
+++ b/test/test_extras.py
@@ -0,0 +1,35 @@
+import requests
+
+
+def test_simple_upscaling_performed(base_url, img2img_basic_image_base64):
+    payload = {
+        "resize_mode": 0,
+        "show_extras_results": True,
+        "gfpgan_visibility": 0,
+        "codeformer_visibility": 0,
+        "codeformer_weight": 0,
+        "upscaling_resize": 2,
+        "upscaling_resize_w": 128,
+        "upscaling_resize_h": 128,
+        "upscaling_crop": True,
+        "upscaler_1": "Lanczos",
+        "upscaler_2": "None",
+        "extras_upscaler_2_visibility": 0,
+        "image": img2img_basic_image_base64,
+    }
+    assert requests.post(f"{base_url}/sdapi/v1/extra-single-image", json=payload).status_code == 200
+
+
+def test_png_info_performed(base_url, img2img_basic_image_base64):
+    payload = {
+        "image": img2img_basic_image_base64,
+    }
+    assert requests.post(f"{base_url}/sdapi/v1/extra-single-image", json=payload).status_code == 200
+
+
+def test_interrogate_performed(base_url, img2img_basic_image_base64):
+    payload = {
+        "image": img2img_basic_image_base64,
+        "model": "clip",
+    }
+    assert requests.post(f"{base_url}/sdapi/v1/extra-single-image", json=payload).status_code == 200
diff --git a/test/test_img2img.py b/test/test_img2img.py
new file mode 100644
index 00000000..117d2d1e
--- /dev/null
+++ b/test/test_img2img.py
@@ -0,0 +1,68 @@
+
+import pytest
+import requests
+
+
+@pytest.fixture()
+def url_img2img(base_url):
+    return f"{base_url}/sdapi/v1/img2img"
+
+
+@pytest.fixture()
+def simple_img2img_request(img2img_basic_image_base64):
+    return {
+        "batch_size": 1,
+        "cfg_scale": 7,
+        "denoising_strength": 0.75,
+        "eta": 0,
+        "height": 64,
+        "include_init_images": False,
+        "init_images": [img2img_basic_image_base64],
+        "inpaint_full_res": False,
+        "inpaint_full_res_padding": 0,
+        "inpainting_fill": 0,
+        "inpainting_mask_invert": False,
+        "mask": None,
+        "mask_blur": 4,
+        "n_iter": 1,
+        "negative_prompt": "",
+        "override_settings": {},
+        "prompt": "example prompt",
+        "resize_mode": 0,
+        "restore_faces": False,
+        "s_churn": 0,
+        "s_noise": 1,
+        "s_tmax": 0,
+        "s_tmin": 0,
+        "sampler_index": "Euler a",
+        "seed": -1,
+        "seed_resize_from_h": -1,
+        "seed_resize_from_w": -1,
+        "steps": 3,
+        "styles": [],
+        "subseed": -1,
+        "subseed_strength": 0,
+        "tiling": False,
+        "width": 64,
+    }
+
+
+def test_img2img_simple_performed(url_img2img, simple_img2img_request):
+    assert requests.post(url_img2img, json=simple_img2img_request).status_code == 200
+
+
+def test_inpainting_masked_performed(url_img2img, simple_img2img_request, mask_basic_image_base64):
+    simple_img2img_request["mask"] = mask_basic_image_base64
+    assert requests.post(url_img2img, json=simple_img2img_request).status_code == 200
+
+
+def test_inpainting_with_inverted_masked_performed(url_img2img, simple_img2img_request, mask_basic_image_base64):
+    simple_img2img_request["mask"] = mask_basic_image_base64
+    simple_img2img_request["inpainting_mask_invert"] = True
+    assert requests.post(url_img2img, json=simple_img2img_request).status_code == 200
+
+
+def test_img2img_sd_upscale_performed(url_img2img, simple_img2img_request):
+    simple_img2img_request["script_name"] = "sd upscale"
+    simple_img2img_request["script_args"] = ["", 8, "Lanczos", 2.0]
+    assert requests.post(url_img2img, json=simple_img2img_request).status_code == 200
diff --git a/test/test_txt2img.py b/test/test_txt2img.py
new file mode 100644
index 00000000..6eb94f0a
--- /dev/null
+++ b/test/test_txt2img.py
@@ -0,0 +1,90 @@
+
+import pytest
+import requests
+
+
+@pytest.fixture()
+def url_txt2img(base_url):
+    return f"{base_url}/sdapi/v1/txt2img"
+
+
+@pytest.fixture()
+def simple_txt2img_request():
+    return {
+        "batch_size": 1,
+        "cfg_scale": 7,
+        "denoising_strength": 0,
+        "enable_hr": False,
+        "eta": 0,
+        "firstphase_height": 0,
+        "firstphase_width": 0,
+        "height": 64,
+        "n_iter": 1,
+        "negative_prompt": "",
+        "prompt": "example prompt",
+        "restore_faces": False,
+        "s_churn": 0,
+        "s_noise": 1,
+        "s_tmax": 0,
+        "s_tmin": 0,
+        "sampler_index": "Euler a",
+        "seed": -1,
+        "seed_resize_from_h": -1,
+        "seed_resize_from_w": -1,
+        "steps": 3,
+        "styles": [],
+        "subseed": -1,
+        "subseed_strength": 0,
+        "tiling": False,
+        "width": 64,
+    }
+
+
+def test_txt2img_simple_performed(url_txt2img, simple_txt2img_request):
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_with_negative_prompt_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["negative_prompt"] = "example negative prompt"
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_with_complex_prompt_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["prompt"] = "((emphasis)), (emphasis1:1.1), [to:1], [from::2], [from:to:0.3], [alt|alt1]"
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_not_square_image_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["height"] = 128
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_with_hrfix_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["enable_hr"] = True
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_with_tiling_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["tiling"] = True
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_with_restore_faces_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["restore_faces"] = True
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+@pytest.mark.parametrize("sampler", ["PLMS", "DDIM", "UniPC"])
+def test_txt2img_with_vanilla_sampler_performed(url_txt2img, simple_txt2img_request, sampler):
+    simple_txt2img_request["sampler_index"] = sampler
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_multiple_batches_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["n_iter"] = 2
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
+
+
+def test_txt2img_batch_performed(url_txt2img, simple_txt2img_request):
+    simple_txt2img_request["batch_size"] = 2
+    assert requests.post(url_txt2img, json=simple_txt2img_request).status_code == 200
diff --git a/test/test_utils.py b/test/test_utils.py
new file mode 100644
index 00000000..edba0b18
--- /dev/null
+++ b/test/test_utils.py
@@ -0,0 +1,33 @@
+import pytest
+import requests
+
+
+def test_options_write(base_url):
+    url_options = f"{base_url}/sdapi/v1/options"
+    response = requests.get(url_options)
+    assert response.status_code == 200
+
+    pre_value = response.json()["send_seed"]
+
+    assert requests.post(url_options, json={'send_seed': (not pre_value)}).status_code == 200
+
+    response = requests.get(url_options)
+    assert response.status_code == 200
+    assert response.json()['send_seed'] == (not pre_value)
+
+    requests.post(url_options, json={"send_seed": pre_value})
+
+
+@pytest.mark.parametrize("url", [
+    "sdapi/v1/cmd-flags",
+    "sdapi/v1/samplers",
+    "sdapi/v1/upscalers",
+    "sdapi/v1/sd-models",
+    "sdapi/v1/hypernetworks",
+    "sdapi/v1/face-restorers",
+    "sdapi/v1/realesrgan-models",
+    "sdapi/v1/prompt-styles",
+    "sdapi/v1/embeddings",
+])
+def test_get_api_url(base_url, url):
+    assert requests.get(f"{base_url}/{url}").status_code == 200