o h@sddlZddlZddlZddlmZmZmZmZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZddlZddlmZddlmZmZddlmZdd lm Z dd l!m"Z"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,e)ryddl-Z-e'rddl.Z.dd l/m0Z1dd l/m2Z3e*4e5Z6GdddeZ7edddZ8edZ9e e9ge fZ:e e ge9fZ;ee:e;fZGdddZ?Gddde?Z@dee7deAde7fddZBdS)N)Fieldasdict dataclass is_dataclass)Path) AnyCallableClassVarDictListOptionalProtocolTupleTypeTypeVarUnion) constants)EntryNotFoundErrorHfHubHTTPError)hf_hub_download)HfApi) ModelCard ModelCardData)SoftTemporaryDirectory is_jsonableis_safetensors_availableis_simple_optional_typeis_torch_availableloggingunwrap_simple_optional_typevalidate_hf_hub_args) load_model) save_modelc@s"eZdZUeeeefed<dS)DataclassInstance__dataclass_fields__N)__name__ __module__ __qualname__r r strr__annotations__r+r+m/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/huggingface_hub/hub_mixin.pyr$(s r$T ModelHubMixin)boundARGS_Ta_ --- # For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 # Doc / guide: https://huggingface.co/docs/hub/model-cards {{ card_data }} --- This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: - Code: {{ repo_url | default("[More Information Needed]", true) }} - Paper: {{ paper_url | default("[More Information Needed]", true) }} - Docs: {{ docs_url | default("[More Information Needed]", true) }} c@sNeZdZUeed<eed<dZeeed<dZeeed<dZ eeed<dS) MixinInfomodel_card_templatemodel_card_dataNdocs_url paper_urlrepo_url) r&r'r(r)r*rr4r r5r6r+r+r+r,r1Cs r1cseZdZUdZdZeeeefe d<e e d<e e d<e e ejfe d<e e efe d<eedfe d <e eefe d <dddeddddddddd d ee d ee dee de deee dee dee dee dee dee deee dee eefddffddZdeedeffdd Zedede fdd Zed!edefd"d#Zed$eededeefd%d&Zddd'dd(d)ee efd*eeeefd+ee d,e d-ee e efdee f d.d/Zd)eddfd0d1Z ee!d'ddddd'dd2deed3ee efd4e d5ee d6ee d7eee e fd8eee efd9e d:ee defd;d<Z"edeed=e d:ee d8eee efd4e d6ee d5ee d9e d7eee e fdefd>d?Z#e!dd@dddddddddA d+e d*eeeefdBe dCee d7ee dDee dEee dFeeee e fdGeeee e fdHeeee e fd-ee e efde fdIdJZ$de%fdKdLZ&Z'S)Mr.a A generic mixin to integrate ANY machine learning framework with the Hub. To integrate your framework, your model class must inherit from this class. Custom logic for saving/loading models have to be overwritten in [`_from_pretrained`] and [`_save_pretrained`]. [`PyTorchModelHubMixin`] is a good example of mixin integration with the Hub. Check out our [integration guide](../guides/integrations) for more instructions. When inheriting from [`ModelHubMixin`], you can define class-level attributes. These attributes are not passed to `__init__` but to the class definition itself. This is useful to define metadata about the library integrating [`ModelHubMixin`]. For more details on how to integrate the mixin with your library, checkout the [integration guide](../guides/integrations). Args: repo_url (`str`, *optional*): URL of the library repository. Used to generate model card. paper_url (`str`, *optional*): URL of the library paper. Used to generate model card. docs_url (`str`, *optional*): URL of the library documentation. Used to generate model card. model_card_template (`str`, *optional*): Template of the model card. Used to generate model card. Defaults to a generic template. language (`str` or `List[str]`, *optional*): Language supported by the library. Used to generate model card. library_name (`str`, *optional*): Name of the library integrating ModelHubMixin. Used to generate model card. license (`str`, *optional*): License of the library integrating ModelHubMixin. Used to generate model card. E.g: "apache-2.0" license_name (`str`, *optional*): Name of the library integrating ModelHubMixin. Used to generate model card. Only used if `license` is set to `other`. E.g: "coqui-public-model-license". license_link (`str`, *optional*): URL to the license of the library integrating ModelHubMixin. Used to generate model card. Only used if `license` is set to `other` and `license_name` is set. E.g: "https://coqui.ai/cpml". pipeline_tag (`str`, *optional*): Tag of the pipeline. Used to generate model card. E.g. "text-classification". tags (`List[str]`, *optional*): Tags to be added to the model card. Used to generate model card. E.g. ["computer-vision"] coders (`Dict[Type, Tuple[Callable, Callable]]`, *optional*): Dictionary of custom types and their encoders/decoders. Used to encode/decode arguments that are not jsonable by default. E.g dataclasses, argparse.Namespace, OmegaConf, etc. Example: ```python >>> from huggingface_hub import ModelHubMixin # Inherit from ModelHubMixin >>> class MyCustomModel( ... ModelHubMixin, ... library_name="my-library", ... tags=["computer-vision"], ... repo_url="https://github.com/huggingface/my-cool-library", ... paper_url="https://arxiv.org/abs/2304.12244", ... docs_url="https://huggingface.co/docs/my-cool-library", ... # ^ optional metadata to generate model card ... ): ... def __init__(self, size: int = 512, device: str = "cpu"): ... # define how to initialize your model ... super().__init__() ... ... ... ... def _save_pretrained(self, save_directory: Path) -> None: ... # define how to serialize your model ... ... ... ... @classmethod ... def from_pretrained( ... cls: Type[T], ... pretrained_model_name_or_path: Union[str, Path], ... *, ... force_download: bool = False, ... resume_download: Optional[bool] = None, ... proxies: Optional[Dict] = None, ... token: Optional[Union[str, bool]] = None, ... cache_dir: Optional[Union[str, Path]] = None, ... local_files_only: bool = False, ... revision: Optional[str] = None, ... **model_kwargs, ... ) -> T: ... # define how to deserialize your model ... ... >>> model = MyCustomModel(size=256, device="gpu") # Save model weights to local directory >>> model.save_pretrained("my-awesome-model") # Push model weights to the Hub >>> model.push_to_hub("my-awesome-model") # Download and initialize weights from the Hub >>> reloaded_model = MyCustomModel.from_pretrained("username/my-awesome-model") >>> reloaded_model.size 256 # Model card has been correctly populated >>> from huggingface_hub import ModelCard >>> card = ModelCard.load("username/my-awesome-model") >>> card.data.tags ["x-custom-tag", "pytorch_model_hub_mixin", "model_hub_mixin"] >>> card.data.library_name "my-library" ``` N_hub_mixin_config_hub_mixin_info_hub_mixin_inject_config_hub_mixin_init_parameters"_hub_mixin_jsonable_default_values. _hub_mixin_jsonable_custom_types_hub_mixin_coders) r6r5r4r2language library_namelicense license_name license_link pipeline_tagtagscodersr6r5r4r2r>r?r@rArBrCrDrEreturnc st| pg} | dt|td} tdr>|tkr#jj| _tdijj | _ jj | _ jj | _ jj | _ | _|durL|tkrL|| _|durS|| _ |durZ|| _ |dura|| _ |duri|| j _|durq|| j _|dury|| j _|dur|| j _| dur| | j _| dur| | j _| dur| j jdur| j j| n| | j _tt| j j| j _| pi_tj_ttjj _!fddj!"D_#dtj$j v_%dS) zIInspect __init__ signature only once when subclassing + handle modelcard.model_hub_mixin)r2r3r8Ncs6i|]}|jtjjur|jr|j|jqSr+)defaultinspect Parameterempty _is_jsonablename _encode_arg.0paramclsr+r, s z3ModelHubMixin.__init_subclass__..configr+)&super__init_subclass__appendr1rhasattrDEFAULT_MODEL_CARDr8r2r3to_dictr4r5r6r>r?r@rArBrCrDextendsortedsetr=tuplekeysr<dictrI signature__init__ parametersr:valuesr;_from_pretrainedr9)rSr6r5r4r2r>r?r@rArBrCrDrEinfo __class__rRr,rWsX          zModelHubMixin.__init_subclass__rScstjdur Siddttjdd|D|}t|dr0|d_Sijfdd| D}| di}t |t rQ| ||ikrX|_S)aACreate a new instance of the class and handle config. 3 cases: - If `self._hub_mixin_config` is already set, do nothing. - If `config` is passed as a dataclass, set it as `self._hub_mixin_config`. - Otherwise, build `self._hub_mixin_config` from default values and passed values. NcSsi|]\}}||qSr+r+rPkeyvaluer+r+r,rT1sz)ModelHubMixin.__new__..rrUcs&i|]\}}|r||qSr+)rLrNrjrSinstancer+r,rTFs  )rV__new__r7ziplistr:rgetr;itemspop isinstanceraupdate)rSargskwargs passed_values init_config passed_configrhrmr,ro!s:       zModelHubMixin.__new__rlcCs$t|rdSt||jrdSt|S)z&Check if a value is JSON serializable.T)rrur<r)rSrlr+r+r,rLWs  zModelHubMixin._is_jsonableargcCsPt|rt|S|jD]\}\}}t||r%|durdS||Sq |S)z3Encode an argument into a JSON serializable format.N)rrr=rsru)rSr|type_encoder_r+r+r,rN`s  zModelHubMixin._encode_arg expected_typecCsjt|r|dur dSt|}t|rt||S|jD]\}\}}t|r2t||r2||Sq|S)z2Decode a JSON serializable value into an argument.N) rr r_load_dataclassr=rsrIisclass issubclass)rSrrlr}rdecoderr+r+r, _decode_argls  zModelHubMixin._decode_argF)rUrepo_id push_to_hubmodel_card_kwargssave_directoryrUrrrc Kst|}|jddd|tj}|jdd|||dur"|j}|dur?t|r.t|}| s?t j |ddd}| ||d} |durI|ni}| s\|j d i||d|r||} |durj|| d<|durq|j}|jd ||d | SdS) ag Save weights in local directory. Args: save_directory (`str` or `Path`): Path to directory in which the model weights and configuration will be saved. config (`dict` or `DataclassInstance`, *optional*): Model configuration specified as a key/value dictionary or a dataclass instance. push_to_hub (`bool`, *optional*, defaults to `False`): Whether or not to push your model to the Huggingface Hub after saving it. repo_id (`str`, *optional*): ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to the folder name if not provided. model_card_kwargs (`Dict[str, Any]`, *optional*): Additional arguments passed to the model card template to customize the model card. push_to_hub_kwargs: Additional key word arguments passed along to the [`~ModelHubMixin.push_to_hub`] method. Returns: `str` or `None`: url of the commit on the Hub if `push_to_hub=True`, `None` otherwise. T)parentsexist_ok) missing_okN) sort_keysindentz README.mdrU)rrr+)rmkdirr CONFIG_NAMEunlink_save_pretrainedr7rrexistsjsondumps write_textgenerate_model_cardsavecopyrMr) selfrrUrrrpush_to_hub_kwargs config_path config_strmodel_card_pathrxr+r+r,save_pretrained}s2    zModelHubMixin.save_pretrainedcCt)aD Overwrite this method in subclass to define how to save your model. Check out our [integration guide](../guides/integrations) for instructions. Args: save_directory (`str` or `Path`): Path to directory in which the model weights and configuration will be saved. NotImplementedError)rrr+r+r,rs zModelHubMixin._save_pretrained)force_downloadresume_downloadproxiestoken cache_dirlocal_files_onlyrevisionpretrained_model_name_or_pathrrrrrrrc Kst|} d} tj| r-tjt| vrtj| tj} n@t tjdt |  n0zt | tj|||||||d } Wnt y\} zttjdt| WYd} ~ nd} ~ wwd} | durt| ddd }t|} Wdn1s{wY| D]\}}||jvr|j|j}|tjjur|||| |<q|jD]}|j| vr|j| vr| |j| |j<qd|jvrd| vr|jdj}||| } | | d<t|r|jD]}|| vr|| vr| || |<qntd d |jDr | D] \}}|| vr || |<q|jrd| vr| | d<|jd t| |||||||d | }| dur@t |d ddifvr@| |_!|S)a Download a model from the Huggingface Hub and instantiate it. Args: pretrained_model_name_or_path (`str`, `Path`): - Either the `model_id` (string) of a model hosted on the Hub, e.g. `bigscience/bloom`. - Or a path to a `directory` containing model weights saved using [`~transformers.PreTrainedModel.save_pretrained`], e.g., `../path/to/my_model_directory/`. revision (`str`, *optional*): Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. Defaults to the latest commit on `main` branch. force_download (`bool`, *optional*, defaults to `False`): Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding the existing cache. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on every request. token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. By default, it will use the token cached when running `huggingface-cli login`. cache_dir (`str`, `Path`, *optional*): Path to the folder where cached files are stored. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, avoid downloading the file and return the path to the local cached file if it exists. model_kwargs (`Dict`, *optional*): Additional kwargs to pass to the model during initialization. Nz not found in rfilenamerrrrrrrz# not found on the HuggingFace Hub: rzutf-8)encodingrUcss|] }|jtjjkVqdS)N)kindrIrJ VAR_KEYWORDrOr+r+r, -sz0ModelHubMixin.from_pretrained..)model_idrrrrrrrr7r+)"r)ospathisdirrrlistdirjoinloggerwarningrresolverrrgopenrloadrsr: annotationrIrJrKrrerMrr%anyr9rfgetattrr7)rSrrrrrrrr model_kwargsr config_fileerUfrkrlrrQconfig_annotationrnr+r+r,from_pretraineds*    &            zModelHubMixin.from_pretrainedrc Kr)aOverwrite this method in subclass to define how to load your model from pretrained. Use [`hf_hub_download`] or [`snapshot_download`] to download files from the Hub before loading them. Most args taken as input can be directly passed to those 2 methods. If needed, you can add more arguments to this method using "model_kwargs". For example [`PyTorchModelHubMixin._from_pretrained`] takes as input a `map_location` parameter to set on which device the model should be loaded. Check out our [integration guide](../guides/integrations) for more instructions. Args: model_id (`str`): ID of the model to load from the Huggingface Hub (e.g. `bigscience/bloom`). revision (`str`, *optional*): Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. Defaults to the latest commit on `main` branch. force_download (`bool`, *optional*, defaults to `False`): Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding the existing cache. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint (e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`). token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. By default, it will use the token cached when running `huggingface-cli login`. cache_dir (`str`, `Path`, *optional*): Path to the folder where cached files are stored. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, avoid downloading the file and return the path to the local cached file if it exists. model_kwargs: Additional keyword arguments passed along to the [`~ModelHubMixin._from_pretrained`] method. r) rSrrrrrrrrrr+r+r,rfIs-zModelHubMixin._from_pretrainedz!Push model using huggingface_hub.) rUcommit_messageprivaterbranch create_prallow_patternsignore_patternsdelete_patternsrrrrrrrrc  Cs~t|d} | j||ddj}t$} t| |}|j||| d| j|d|||||| | d WdS1s8wYdS)a= Upload model checkpoint to the Hub. Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more details. Args: repo_id (`str`): ID of the repository to push to (example: `"username/my-model"`). config (`dict` or `DataclassInstance`, *optional*): Model configuration specified as a key/value dictionary or a dataclass instance. commit_message (`str`, *optional*): Message to commit while pushing. private (`bool`, *optional*): Whether the repository created should be private. If `None` (default), the repo will be public unless the organization's default is private. token (`str`, *optional*): The token to use as HTTP bearer authorization for remote files. By default, it will use the token cached when running `huggingface-cli login`. branch (`str`, *optional*): The git branch on which to push the model. This defaults to `"main"`. create_pr (`boolean`, *optional*): Whether or not to create a Pull Request from `branch` with that commit. Defaults to `False`. allow_patterns (`List[str]` or `str`, *optional*): If provided, only files matching at least one pattern are pushed. ignore_patterns (`List[str]` or `str`, *optional*): If provided, files matching any of the patterns are not pushed. delete_patterns (`List[str]` or `str`, *optional*): If provided, remote files matching any of the patterns will be deleted from the repo. model_card_kwargs (`Dict[str, Any]`, *optional*): Additional arguments passed to the model card template to customize the model card. Returns: The url of the commit of your model in the given repository. )rT)rrr)rUrmodel) r repo_type folder_pathrrrrrrN)r create_reporrrr upload_folder)rrrUrrrrrrrrrapitmp saved_pathr+r+r,rxs" 4 $zModelHubMixin.push_to_hubcOs4tjd|jj|jj|jj|jj|jjd|}|S)N) card_data template_strr6r5r4r+)r from_templater8r3r2r6r5r4)rrwrxcardr+r+r,rsz!ModelHubMixin.generate_model_card)(r&r'r(__doc__r7r rrar$r*r1boolr r)rIrJrrrCODER_TrZr rWr-ro classmethodrLrNr0rrrrr!rrfrrr __classcell__r+r+rhr,r.LsH m      \6   D     {   .      FcseZdZdZdddeeeddffddZdeddfd d Z e d d d dedeedee eefde dee dee de de ee dfdede fddZe dededede def ddZe dededede def ddZZS) PyTorchModelHubMixina~ Implementation of [`ModelHubMixin`] to provide model Hub upload/download capabilities to PyTorch models. The model is set in evaluation mode by default using `model.eval()` (dropout modules are deactivated). To train the model, you should first set it back in training mode with `model.train()`. See [`ModelHubMixin`] for more details on how to use the mixin. Example: ```python >>> import torch >>> import torch.nn as nn >>> from huggingface_hub import PyTorchModelHubMixin >>> class MyModel( ... nn.Module, ... PyTorchModelHubMixin, ... library_name="keras-nlp", ... repo_url="https://github.com/keras-team/keras-nlp", ... paper_url="https://arxiv.org/abs/2304.12244", ... docs_url="https://keras.io/keras_nlp/", ... # ^ optional metadata to generate model card ... ): ... def __init__(self, hidden_size: int = 512, vocab_size: int = 30000, output_size: int = 4): ... super().__init__() ... self.param = nn.Parameter(torch.rand(hidden_size, vocab_size)) ... self.linear = nn.Linear(output_size, vocab_size) ... def forward(self, x): ... return self.linear(x + self.param) >>> model = MyModel(hidden_size=256) # Save model weights to local directory >>> model.save_pretrained("my-awesome-model") # Push model weights to the Hub >>> model.push_to_hub("my-awesome-model") # Download and initialize weights from the Hub >>> model = MyModel.from_pretrained("username/my-awesome-model") >>> model.hidden_size 256 ``` N)rDrDrFcs,|pg}|d||d<tj|i|S)Npytorch_model_hub_mixinrD)rXrVrW)rSrDrwrxrhr+r,rWs z&PyTorchModelHubMixin.__init_subclass__rcCs,t|dr|jn|}t|t|tjdS)z7Save weights from a Pytorch model to a local directory.moduleN)rYrsave_model_as_safetensorr)rSAFETENSORS_SINGLE_FILE)rr model_to_saver+r+r,rsz%PyTorchModelHubMixin._save_pretrainedcpuF) map_locationstrictrrrrrrrrrrc Ks|di| } tj|r!tdtj|tj} || | | | Szt|tj|||||||d } || | | | WSt yWt|tj |||||||d } | | | | | YSw)z.Nr+)rs)rrr+rr,rNs r)CrIrr dataclassesrrrrpathlibrtypingrrr r r r r rrrrpackaging.versionrrerrorsrr file_downloadrhf_apirrepocardrrutilsrrrrrrr r!rrsafetensors.torchr"rr#r get_loggerr&rr$r-r0 ENCODER_T DECODER_TrrZr1r.rrarr+r+r+r,sL 4   (