From 9e02812afd10582f00a7fbbfa63c8f9188678e26 Mon Sep 17 00:00:00 2001 From: arcticfaded Date: Mon, 17 Oct 2022 07:02:08 +0000 Subject: pydantic instrumentation --- modules/api/processing.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 modules/api/processing.py (limited to 'modules/api/processing.py') diff --git a/modules/api/processing.py b/modules/api/processing.py new file mode 100644 index 00000000..459a8f49 --- /dev/null +++ b/modules/api/processing.py @@ -0,0 +1,99 @@ +from inflection import underscore +from typing import Any, Dict, Optional +from pydantic import BaseModel, Field, create_model +from modules.processing import StableDiffusionProcessing, Processed, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images +import inspect + + +class ModelDef(BaseModel): + """Assistance Class for Pydantic Dynamic Model Generation""" + + field: str + field_alias: str + field_type: Any + field_value: Any + + +class pydanticModelGenerator: + """ + Takes source_data:Dict ( a single instance example of something like a JSON node) and self generates a pythonic data model with Alias to original source field names. This makes it easy to popuate or export to other systems yet handle the data in a pythonic way. + Being a pydantic datamodel all the richness of pydantic data validation is available and these models can easily be used in FastAPI and or a ORM + + It does not process full JSON data structures but takes simple JSON document with basic elements + + Provide a model_name, an example of JSON data and a dict of type overrides + + Example: + + source_data = {'Name': '48 Rainbow Rd', + 'GroupAddressStyle': 'ThreeLevel', + 'LastModified': '2020-12-21T07:02:51.2400232Z', + 'ProjectStart': '2020-12-03T07:36:03.324856Z', + 'Comment': '', + 'CompletionStatus': 'Editing', + 'LastUsedPuid': '955', + 'Guid': '0c85957b-c2ae-4985-9752-b300ab385b36'} + + source_overrides = {'Guid':{'type':uuid.UUID}, + 'LastModified':{'type':datetime }, + 'ProjectStart':{'type':datetime }, + } + source_optionals = {"Comment":True} + + #create Model + model_Project=pydanticModelGenerator( + model_name="Project", + source_data=source_data, + overrides=source_overrides, + optionals=source_optionals).generate_model() + + #create instance using DynamicModel + project_instance=model_Project(**project_info) + + """ + + def __init__( + self, + model_name: str = None, + source_data: str = None, + params: Dict = {}, + overrides: Dict = {}, + optionals: Dict = {}, + ): + def field_type_generator(k, v, overrides, optionals): + print(k, v) + field_type = str if not overrides.get(k) else overrides[k]["type"] + if v is None: + field_type = Any + else: + field_type = type(v) + + return Optional[field_type] + + self._model_name = model_name + self._json_data = source_data + self._model_def = [ + ModelDef( + field=underscore(k), + field_alias=k, + field_type=field_type_generator(k, v, overrides, optionals), + field_value=v + ) + for (k,v) in source_data.items() if k in params + ] + + def generate_model(self): + """ + Creates a pydantic BaseModel + from the json and overrides provided at initialization + """ + fields = { + d.field: (d.field_type, Field(default=d.field_value, alias=d.field_alias)) for d in self._model_def + } + DynamicModel = create_model(self._model_name, **fields) + DynamicModel.__config__.allow_population_by_field_name = True + return DynamicModel + +StableDiffusionProcessingAPI = pydanticModelGenerator("StableDiffusionProcessing", + StableDiffusionProcessing().__dict__, + inspect.signature(StableDiffusionProcessing.__init__).parameters).generate_model() \ No newline at end of file -- cgit v1.2.1 From 832b490e5173f78c4d3aa7ca9ca9ac794d140664 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 17 Oct 2022 03:18:41 -0400 Subject: Update processing.py --- modules/api/processing.py | 41 +++++------------------------------------ 1 file changed, 5 insertions(+), 36 deletions(-) (limited to 'modules/api/processing.py') diff --git a/modules/api/processing.py b/modules/api/processing.py index 459a8f49..4c3d0bd0 100644 --- a/modules/api/processing.py +++ b/modules/api/processing.py @@ -16,46 +16,15 @@ class ModelDef(BaseModel): class pydanticModelGenerator: """ - Takes source_data:Dict ( a single instance example of something like a JSON node) and self generates a pythonic data model with Alias to original source field names. This makes it easy to popuate or export to other systems yet handle the data in a pythonic way. - Being a pydantic datamodel all the richness of pydantic data validation is available and these models can easily be used in FastAPI and or a ORM - - It does not process full JSON data structures but takes simple JSON document with basic elements - - Provide a model_name, an example of JSON data and a dict of type overrides - - Example: - - source_data = {'Name': '48 Rainbow Rd', - 'GroupAddressStyle': 'ThreeLevel', - 'LastModified': '2020-12-21T07:02:51.2400232Z', - 'ProjectStart': '2020-12-03T07:36:03.324856Z', - 'Comment': '', - 'CompletionStatus': 'Editing', - 'LastUsedPuid': '955', - 'Guid': '0c85957b-c2ae-4985-9752-b300ab385b36'} - - source_overrides = {'Guid':{'type':uuid.UUID}, - 'LastModified':{'type':datetime }, - 'ProjectStart':{'type':datetime }, - } - source_optionals = {"Comment":True} - - #create Model - model_Project=pydanticModelGenerator( - model_name="Project", - source_data=source_data, - overrides=source_overrides, - optionals=source_optionals).generate_model() - - #create instance using DynamicModel - project_instance=model_Project(**project_info) - + Takes in created classes and stubs them out in a way FastAPI/Pydantic is happy about: + source_data is a snapshot of the default values produced by the class + params are the names of the actual keys required by __init__ """ def __init__( self, model_name: str = None, - source_data: str = None, + source_data: {} = {}, params: Dict = {}, overrides: Dict = {}, optionals: Dict = {}, @@ -96,4 +65,4 @@ class pydanticModelGenerator: StableDiffusionProcessingAPI = pydanticModelGenerator("StableDiffusionProcessing", StableDiffusionProcessing().__dict__, - inspect.signature(StableDiffusionProcessing.__init__).parameters).generate_model() \ No newline at end of file + inspect.signature(StableDiffusionProcessing.__init__).parameters).generate_model() -- cgit v1.2.1 From 99013ba68a5fe1bde3621632e5539c03562a3ae8 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 17 Oct 2022 03:20:17 -0400 Subject: Update processing.py --- modules/api/processing.py | 1 - 1 file changed, 1 deletion(-) (limited to 'modules/api/processing.py') diff --git a/modules/api/processing.py b/modules/api/processing.py index 4c3d0bd0..e4df93c5 100644 --- a/modules/api/processing.py +++ b/modules/api/processing.py @@ -30,7 +30,6 @@ class pydanticModelGenerator: optionals: Dict = {}, ): def field_type_generator(k, v, overrides, optionals): - print(k, v) field_type = str if not overrides.get(k) else overrides[k]["type"] if v is None: field_type = Any -- cgit v1.2.1 From f80e914ac4aa69a9783b4040813253500b34d925 Mon Sep 17 00:00:00 2001 From: arcticfaded Date: Mon, 17 Oct 2022 19:10:36 +0000 Subject: example API working with gradio --- modules/api/processing.py | 56 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 18 deletions(-) (limited to 'modules/api/processing.py') diff --git a/modules/api/processing.py b/modules/api/processing.py index e4df93c5..b6798241 100644 --- a/modules/api/processing.py +++ b/modules/api/processing.py @@ -5,6 +5,24 @@ from modules.processing import StableDiffusionProcessing, Processed, StableDiffu import inspect +API_NOT_ALLOWED = [ + "self", + "kwargs", + "sd_model", + "outpath_samples", + "outpath_grids", + "sampler_index", + "do_not_save_samples", + "do_not_save_grid", + "extra_generation_params", + "overlay_images", + "do_not_reload_embeddings", + "seed_enable_extras", + "prompt_for_display", + "sampler_noise_scheduler_override", + "ddim_discretize" +] + class ModelDef(BaseModel): """Assistance Class for Pydantic Dynamic Model Generation""" @@ -14,7 +32,7 @@ class ModelDef(BaseModel): field_value: Any -class pydanticModelGenerator: +class PydanticModelGenerator: """ Takes in created classes and stubs them out in a way FastAPI/Pydantic is happy about: source_data is a snapshot of the default values produced by the class @@ -24,30 +42,33 @@ class pydanticModelGenerator: def __init__( self, model_name: str = None, - source_data: {} = {}, - params: Dict = {}, - overrides: Dict = {}, - optionals: Dict = {}, + class_instance = None ): - def field_type_generator(k, v, overrides, optionals): - field_type = str if not overrides.get(k) else overrides[k]["type"] - if v is None: - field_type = Any - else: - field_type = type(v) + def field_type_generator(k, v): + # field_type = str if not overrides.get(k) else overrides[k]["type"] + # print(k, v.annotation, v.default) + field_type = v.annotation return Optional[field_type] + def merge_class_params(class_): + all_classes = list(filter(lambda x: x is not object, inspect.getmro(class_))) + parameters = {} + for classes in all_classes: + parameters = {**parameters, **inspect.signature(classes.__init__).parameters} + return parameters + + self._model_name = model_name - self._json_data = source_data + self._class_data = merge_class_params(class_instance) self._model_def = [ ModelDef( field=underscore(k), field_alias=k, - field_type=field_type_generator(k, v, overrides, optionals), - field_value=v + field_type=field_type_generator(k, v), + field_value=v.default ) - for (k,v) in source_data.items() if k in params + for (k,v) in self._class_data.items() if k not in API_NOT_ALLOWED ] def generate_model(self): @@ -60,8 +81,7 @@ class pydanticModelGenerator: } DynamicModel = create_model(self._model_name, **fields) DynamicModel.__config__.allow_population_by_field_name = True + DynamicModel.__config__.allow_mutation = True return DynamicModel -StableDiffusionProcessingAPI = pydanticModelGenerator("StableDiffusionProcessing", - StableDiffusionProcessing().__dict__, - inspect.signature(StableDiffusionProcessing.__init__).parameters).generate_model() +StableDiffusionProcessingAPI = PydanticModelGenerator("StableDiffusionProcessingTxt2Img", StableDiffusionProcessingTxt2Img).generate_model() -- cgit v1.2.1 From e7f4808505f7a6339927c32b9a0c01bc9134bdeb Mon Sep 17 00:00:00 2001 From: arcticfaded Date: Tue, 18 Oct 2022 19:04:56 +0000 Subject: provide sampler by name --- modules/api/processing.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'modules/api/processing.py') diff --git a/modules/api/processing.py b/modules/api/processing.py index b6798241..2e6483ee 100644 --- a/modules/api/processing.py +++ b/modules/api/processing.py @@ -42,7 +42,8 @@ class PydanticModelGenerator: def __init__( self, model_name: str = None, - class_instance = None + class_instance = None, + additional_fields = None, ): def field_type_generator(k, v): # field_type = str if not overrides.get(k) else overrides[k]["type"] @@ -70,6 +71,13 @@ class PydanticModelGenerator: ) for (k,v) in self._class_data.items() if k not in API_NOT_ALLOWED ] + + for fields in additional_fields: + self._model_def.append(ModelDef( + field=underscore(fields["key"]), + field_alias=fields["key"], + field_type=fields["type"], + field_value=fields["default"])) def generate_model(self): """ @@ -84,4 +92,8 @@ class PydanticModelGenerator: DynamicModel.__config__.allow_mutation = True return DynamicModel -StableDiffusionProcessingAPI = PydanticModelGenerator("StableDiffusionProcessingTxt2Img", StableDiffusionProcessingTxt2Img).generate_model() +StableDiffusionProcessingAPI = PydanticModelGenerator( + "StableDiffusionProcessingTxt2Img", + StableDiffusionProcessingTxt2Img, + [{"key": "sampler_index", "type": str, "default": "k_euler_a"}] +).generate_model() -- cgit v1.2.1 From 0f0d6ab8e06898ce066251fc769fe14e77e98ced Mon Sep 17 00:00:00 2001 From: arcticfaded Date: Wed, 19 Oct 2022 05:19:01 +0000 Subject: call sampler by name --- modules/api/processing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'modules/api/processing.py') diff --git a/modules/api/processing.py b/modules/api/processing.py index 2e6483ee..4c541241 100644 --- a/modules/api/processing.py +++ b/modules/api/processing.py @@ -1,7 +1,7 @@ from inflection import underscore from typing import Any, Dict, Optional from pydantic import BaseModel, Field, create_model -from modules.processing import StableDiffusionProcessing, Processed, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images +from modules.processing import StableDiffusionProcessingTxt2Img import inspect @@ -95,5 +95,5 @@ class PydanticModelGenerator: StableDiffusionProcessingAPI = PydanticModelGenerator( "StableDiffusionProcessingTxt2Img", StableDiffusionProcessingTxt2Img, - [{"key": "sampler_index", "type": str, "default": "k_euler_a"}] -).generate_model() + [{"key": "sampler_index", "type": str, "default": "Euler"}] +).generate_model() \ No newline at end of file -- cgit v1.2.1