o hD @s$dZddlZddlmZmZddlmZddlmZddl m Z m Z m Z m Z mZmZmZddlmZmZmZmZd d lmZd d lmZzdd lmZdd lmZddlmZdZ Wn e!yedZ YnwedZ"de de fddZ#dZ$GdddeZ%d8dedee"fddZ&e# d8dede e'dee"de e'fdd Z(e#d9d!e'd"e)de)fd#d$Z* d8d%e ede e'dee"de fd&d'Z+ d8dede e'dee"de e'fd(d)Z,d9d!e'd"e)de)fd*d+Z-ded,e e'de'fd-d.Z.d/e'de e'fd0d1Z/d2e$d3e$d40Z1d5ede fd6d7Z2dS):asContains command to delete some revisions from the HF cache directory. Usage: huggingface-cli delete-cache huggingface-cli delete-cache --disable-tui huggingface-cli delete-cache --dir ~/.cache/huggingface/hub huggingface-cli delete-cache --sort=size NOTE: This command is based on `InquirerPy` to build the multiselect menu in the terminal. This dependency has to be installed with `pip install huggingface_hub[cli]`. Since we want to avoid as much as possible cross-platform issues, I chose a library that is built on top of `python-prompt-toolkit` which seems to be a reference in terminal GUI (actively maintained on both Unix and Windows, 7.9k stars). For the moment, the TUI feature is in beta. See: - https://github.com/kazhala/InquirerPy - https://inquirerpy.readthedocs.io/en/latest/ - https://github.com/prompt-toolkit/python-prompt-toolkit Other solutions could have been: - `simple_term_menu`: would be good as well for our use case but some issues suggest that Windows is less supported. See: https://github.com/IngoMeyer441/simple-term-menu - `PyInquirer`: very similar to `InquirerPy` but older and not maintained anymore. In particular, no support of Python3.10. See: https://github.com/CITGuru/PyInquirer - `pick` (or `pickpack`): easy to use and flexible but built on top of Python's standard library `curses` that is specific to Unix (not implemented on Windows). See https://github.com/wong2/pick and https://github.com/anafvana/pickpack. - `inquirer`: lot of traction (700 stars) but explicitly states "experimental support of Windows". Not built on top of `python-prompt-toolkit`. See https://github.com/magmax/python-inquirer TODO: add support for `huggingface-cli delete-cache aaaaaa bbbbbb cccccc (...)` ? TODO: add "--keep-last" arg to delete revisions that are not on `main` ref TODO: add "--filter" arg to filter repositories by name ? TODO: add "--limit" arg to limit to X repos ? TODO: add "-y" arg for immediate deletion ? See discussions in https://github.com/huggingface/huggingface_hub/issues/1025. N) Namespace_SubParsersActionwraps)mkstemp)AnyCallableIterableListLiteralOptionalUnion)CachedRepoInfoCachedRevisionInfo HFCacheInfoscan_cache_dir)BaseHuggingfaceCLICommand)ANSI)inquirer)Choice) SeparatorTF alphabetical lastUpdatedlastUsedsizefnreturncstfdd}|S)z4Decorator to flag methods that require `InquirerPy`.cststd|i|S)NzThe `delete-cache` command requires extra dependencies to work with the TUI. Please run `pip install huggingface_hub[cli]` to install them. Otherwise, disable TUI using the `--disable-tui` flag.)_inquirer_py_available ImportError)argskwargsry/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/huggingface_hub/commands/delete_cache.py_innerVs z#require_inquirer_py.._innerr)rr'r%r$r&require_inquirer_pyRs r(CANCEL_DELETIONc@s8eZdZedefddZdeddfddZd d ZdS) DeleteCacheCommandparsercCsV|jddd}|jdtddd|jddd d |jd d gd dd|jtddS)Nz delete-cachez*Delete revisions from the cache directory.)helpz--dirzEcache directory (optional). Default to the default HuggingFace cache.)typedefaultr,z --disable-tui store_truezrDisable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu.)actionr,z--sort?rzSort repositories by the specified criteria. Options: 'alphabetical' (A-Z), 'lastUpdated' (newest first), 'lastUsed' (most recent first), 'size' (largest first).)nargschoicesr,)func) add_parser add_argumentstr set_defaultsr*)r+delete_cache_parserr%r%r&register_subcommandis&  z&DeleteCacheCommand.register_subcommandr"rNcCs|j|_|j|_|j|_dSN)dir cache_dir disable_tuisortsort_by)selfr"r%r%r&__init__s zDeleteCacheCommand.__init__cCst|j}|jrt|g|jd}nt|g|jd}t|dkr\t|vr\t||d}|jr2t |}nt |}|r\|j |}t d| t dt|jdt|jd|jdd St d d S) z/Run `delete-cache` command with or without TUI.) preselectedr@rz Confirm deletion ?zStart deletion.zDone. Deleted z repo(s) and z revision(s) for a total of .Nz"Deletion is cancelled. Do nothing.)rr=r>_manual_review_no_tuir@_manual_review_tuilen_CANCEL_DELETION_STR_get_expectations_str_ask_for_confirmation_no_tui_ask_for_confirmation_tuidelete_revisionsprintexecuterepos snapshotsexpected_freed_size_str)rA hf_cache_infoselected_hashesconfirm_message confirmedstrategyr%r%r&runs,    zDeleteCacheCommand.run) __name__ __module__ __qualname__ staticmethodrr:rrBrWr%r%r%r&r*hs " r*repor@cCsb|dkr |j|jfS|dkrtdd|jD S|dkr#|j S|dkr+|j S|j|jfS)Nrrcss|]}|jVqdSr; last_modified).0revr%r%r& sz(_get_repo_sorting_key..rr) repo_typerepo_idlowermax revisions last_accessed size_on_disk)r\r@r%r%r&_get_repo_sorting_keys rirRrCc stj||d}tjd|ddtdd|Dddd d d dfdd }jdd|izWStyAgYSw)zzAsk the user for a manual review of the revisions to delete. Displays a multi-select menu in the terminal (TUI). )rOrCr@zSelect revisions to delete:FdcSs"g|] }t|tr|jr|jqSr%) isinstancerenabledvalue)r_cr%r%r& s"z&_manual_review_tui..rSzWPress to select, to validate and to quit without modification.cSst|dS)Nz revision(s) selected.rG)resultr%r%r&sz$_manual_review_tui..)messager3cycleheight instructionlong_instruction transformerrNcs tddjjDd_dS)NcSsg|] }|dr|dqS)rlrmr%)r_choicer%r%r&rozD_manual_review_tui.._update_expectations..rp)rIcontent_controlr3 _instruction)_checkboxrRr%r&_update_expectationss z0_manual_review_tui.._update_expectationstoggler4)rN) _get_tui_choices_from_scanrOrrrIkb_func_lookupappendrNKeyboardInterrupt)rRrCr@r3rr%rr&rFs0    rFrtr.cCstj||dS)z$Ask for confirmation using Inquirer.r.)rconfirmrN)rtr.r%r%r&rKsrKrOc sg}|ttdddt|fddd}|D]J}|td|jd|jd |jd |j d t|j t dD]&}|t|j |j d d dd t|jpSdd|j|j |vdq;q|S)aBuild a list of choices from the scanned repos. Args: repos (*Iterable[`CachedRepoInfo`]*): List of scanned repos on which we want to delete revisions. preselected (*List[`str`]*): List of revision hashes that will be preselected. sort_by (*Optional[SortingOption_T]*): Sorting direction. Choices: "alphabetical", "lastUpdated", "lastUsed", "size". Return: The list of choices to pass to `inquirer.checkbox`. z=None of the following (if selected, nothing will be deleted).F)namerlc t|Sr;rir\r@r%r&rs z,_get_tui_choices_from_scan..key   (, used )Nz: , (detached) # modified )rrrHsortedrrb capitalizercsize_on_disk_strlast_accessed_strrf_revision_sorting_order commit_hashjoinrefslast_modified_str)rOrCr@r3 sorted_reposr\revisionr%rr&rs> rc sptdd\}}t|g}t|jfddd}|D]E}|d|jd|jd|j d |j d t|j t dD]#}||j |vrGd nd d |j ddt|jpWdd|jq.rz # rrrr#z z # Refs: rrrwrNa* TUI is disabled. In order to select which revisions you want to delete, please edit the following file using the text editor of your choice. Instructions for manual editing are located at the beginning of the file. Edit the file, save it and confirm to continue. File to edit: z css|]}|VqdSr;stripr_liner%r%r&raesz(_manual_review_no_tui..Tz Continue ?Fr)roscloserrOrrbrrcrrrfrrrrropenwrite"_MANUAL_REVIEW_NO_TUI_INSTRUCTIONSrboldrMrsplit_read_manual_review_tmp_filerJrIremove) rRrCr@fdtmp_pathlinesrr\rf instructionsrSr%rr&rE7sN  "  rEcCsld}d}d}|||f}||rdnd} t|}||kr"|S||vr(dS||vr.dStd|q) z'Ask for confirmation using pure-python.)yyes1)nno0rz (Y/n) z (y/N) TFzInvalid input. Must be one of )inputrdrM)rtr.YESNODEFAULTALL full_messageanswerr%r%r&rJus rJrScCs,t|vrdS|j|}t|d|jdS)zFormat a string to display to the user how much space would be saved. Example: ``` >>> _get_expectations_str(hf_cache_info, selected_hashes) '7 revisions selected counting for 4.3G.' ``` zNothing will be deleted.z! revisions selected counting for rD)rHrLrGrQ)rRrSrVr%r%r&rIs  rIrcCsnt| }|}Wdn1swYdd|dD}dd|D}dd|D}dd|DS)aRead the manually reviewed instruction file and return a list of revision hash. Example: ```txt # This is the tmp file content ### # Commented out line 123456789 # revision hash # Something else # a_newer_hash # 2 days ago an_older_hash # 3 days ago ``` ```py >>> _read_manual_review_tmp_file(tmp_path) ['123456789', 'an_older_hash'] ``` NcSsg|]}|qSr%rrr%r%r&rosz0_read_manual_review_tmp_file..rcSsg|] }|ds|qS)r) startswithrr%r%r&roscSsg|] }|ddqS)rr)rrrr%r%r&roscSsg|] }t|dkr|qS)rrq)r_hashr%r%r&ror{)rreadr)rrcontentrselected_linesrSr%r%r&rs  ra # INSTRUCTIONS # ------------ # This is a temporary file created by running `huggingface-cli delete-cache` with the # `--disable-tui` option. It contains a set of revisions that can be deleted from your # local cache directory. # # Please manually review the revisions you want to delete: # - Revision hashes can be commented out with '#'. # - Only non-commented revisions in this file will be deleted. # - Revision hashes that are removed from this file are ignored as well. # - If `aa` line is uncommented, the all cache deletion is cancelled and # no changes will be applied. # # Once you've manually reviewed this file, please confirm deletion in the terminal. This # file will be automatically removed once done. # ------------ # KILL SWITCH # ------------ # Un-comment following line to completely cancel the deletion process # z, # ------------ # REVISIONS # ------------ rcCs|jSr;r])rr%r%r&rsrr;)T)3__doc__rargparserr functoolsrtempfilertypingrrr r r r r utilsrrrrrr _cli_utilsr InquirerPyrInquirerPy.base.controlrInquirerPy.separatorrr r!SortingOption_Tr(rHr*rir7rFboolrKrrErJrIrrrrr%r%r%r&s,  $      N  3 = >%