add pkgs

3rds/ccache.conf

@@ -0,0 +1,5 @@
+base_dir = /home/agent/workspace/
+cache_dir = /home/workspace/dep_libs/ccache
+hash_dir = false
+max_size = 50.0G
+max_files = 0

3rds/pt201.txt

@@ -0,0 +1,288 @@
+setuptools>=45
+absl-py==1.4.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+antlr4-python3-runtime==4.8
+appdirs==1.4.4
+astroid==2.15.5
+async-timeout==4.0.2
+attrs==23.1.0
+audioread==3.0.0
+Babel==2.12.1
+beautifulsoup4==4.12.2
+-e git+https://github.com/pytorch/benchmark.git@e5bf5f0ce01ac7e489e8543ad683b07da7f93d4c#egg=bert_pytorch&subdirectory=torchbenchmark/models/BERT_pytorch
+bitarray==2.7.5
+black==21.4b2
+blinker==1.6.2
+blis==0.7.9
+bokeh==3.1.1
+boto3==1.26.155
+botocore==1.29.155
+braceexpand==0.1.7
+Brotli==1.0.9
+# bs4==0.0.1
+cachetools==5.3.1
+cardboardlint==1.3.1
+catalogue==2.0.8
+certifi==2022.12.7
+cffi==1.15.1
+charset-normalizer==2.1.1
+click==8.1.3
+clip-anytorch==2.5.2
+clldutils==3.19.0
+cloudpickle==2.2.1
+CoCa-pytorch==0.0.7
+colorama==0.4.6
+coloredlogs==15.0.1
+colorlog==6.7.0
+confection==0.0.4
+contourpy==1.1.0
+cssselect2==0.7.0
+csvw==3.1.3
+cycler==0.11.0
+cymem==2.0.7
+Cython==0.29.35
+datasets==2.13.0
+de-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.5.0/de_core_news_sm-3.5.0-py3-none-any.whl#sha256=c9c06808be2f1e7f448b37f94391a50dab2e2f72f9b5c2e979be18aebac4a06c
+decorator==5.1.1
+defusedxml==0.7.1
+detectron2 @ git+https://github.com/facebookresearch/detectron2.git@c470ca31d2b54d038b9e78f48d1d8e1a8debcc68
+dgl==1.1.0
+# diffusers==0.17.1
+dill==0.3.6
+distro==1.8.0
+dlinfo==1.2.1
+docker-pycreds==0.4.0
+dominate==2.8.0
+effdet @ git+https://github.com/rwightman/efficientdet-pytorch.git@79d26d8982b9f8e1f27d9f896e38012ac250fd26
+einops==0.6.1
+ema-pytorch==0.2.3
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl#sha256=0964370218b7e1672a30ac50d72cdc6b16f7c867496f1d60925691188f4d2510
+exceptiongroup==1.1.1
+FastNLP==0.6.0
+fbgemm-gpu-nightly
+ffmpeg-python==0.2.0
+filelock==3.9.0
+Flask==2.3.2
+flatbuffers==23.5.26
+fonttools==4.40.0
+frozenlist==1.3.3
+fsspec==2023.6.0
+ftfy==6.1.1
+functorch @ git+https://github.com/pytorch/functorch.git@b71aa0b4387b86c278132209b99538be48ef4c74
+future==0.18.3
+fvcore==0.1.5.post20221221
+gdown==4.7.1
+gitdb==4.0.10
+GitPython==3.1.31
+google-auth==2.20.0
+google-auth-oauthlib==1.0.0
+grpcio==1.54.2
+gym==0.25.2
+gym-notices==0.0.8
+h5py==3.8.0
+higher==0.2.1
+html5lib==1.1
+# huggingface-hub==0.15.1
+humanfriendly==10.0
+hydra-core==1.1.2
+idna==3.4
+imageio==2.31.1
+importlib-metadata==6.7.0
+importlib-resources==5.2.3
+inflect==6.0.4
+iniconfig==2.0.0
+iopath==0.1.9
+isodate==0.6.1
+isort==5.12.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.2.0
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.17.3
+kaldi-io==0.9.8
+kiwisolver==1.4.4
+kornia @ git+https://github.com/kornia/kornia.git@b7050c39f4c7bf44322acda2bc82883fe918a912
+lameenc==1.4.2
+langcodes==3.3.0
+langdetect==1.0.9
+language-tags==1.2.0
+lazy-object-proxy==1.9.0
+lazy_loader==0.2
+libcst==1.0.1
+librosa==0.9.2
+llvmlite==0.40.1rc1
+lpips==0.1.4
+lxml==4.9.2
+Markdown==3.4.3
+markdown-it-py==3.0.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mccabe==0.7.0
+mdurl==0.1.2
+MonkeyType==23.3.0
+mplcursors==0.5.2
+mpmath==1.2.1
+multidict==6.0.4
+multiprocess==0.70.14
+murmurhash==1.0.9
+musdb==0.4.0
+museval==0.4.1
+mypy-extensions==1.0.0
+networkx==3.0
+nose==1.3.7
+numba==0.57.0
+numpy==1.24.3
+oauthlib==3.2.2
+omegaconf==2.1.1
+onnx==1.14.0
+onnxruntime==1.15.1
+onnxsim==0.4.31
+opacus==1.4.0
+open-clip-torch==2.20.0
+opencv-python==4.7.0.72
+opt-einsum==3.3.0
+optimum==1.8.8
+packaging==23.1
+pandas==2.0.2
+patch==1.16
+pathspec==0.11.1
+pathtools==0.1.2
+pathy==0.10.1
+phonemizer==3.2.1
+Pillow==9.3.0
+pkgutil_resolve_name==1.3.10
+platformdirs==3.6.0
+pluggy==1.0.0
+pooch==1.7.0
+portalocker==2.7.0
+preshed==3.0.8
+prettytable==3.8.0
+protobuf==3.20.3
+psutil==5.9.5
+py-cpuinfo==9.0.0
+pyaml==23.5.9
+pyarrow==12.0.1
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pyclipper==1.3.0.post4
+pycocotools==2.0.6
+pycparser==2.21
+pydantic==1.10.9
+pydot==1.4.2
+pydyf==0.6.0
+pygame==2.1.2
+Pygments==2.15.1
+pylatexenc==2.10
+pylint==2.17.4
+pynvml==11.5.0
+pyparsing==3.1.0
+pypdfium2==3.21.1
+pyphen==0.14.0
+pyrsistent==0.19.3
+pysbd==0.3.4
+PySocks==1.7.1
+pytest==7.3.2
+pytest-benchmark==4.0.0
+python-dateutil==2.8.2
+python-doctr @ git+https://github.com/mindee/doctr.git@acb9f64b11ebad8e53ac60737fcde8dbd3158a22
+pytorch-transformers==1.2.0
+pytorch-warmup==0.1.1
+pytz==2023.3
+PyWavelets==1.4.1
+pyproject-toml
+# pyworld==0.3.3
+pyworld==0.3.4
+PyYAML
+# PyYAML==6.0
+# pyproject.toml-based
+rapidfuzz==2.15.1
+rdflib==6.3.2
+regex==2023.6.3
+requests==2.28.1
+requests-oauthlib==1.3.1
+resampy==0.4.2
+resize-right==0.0.2
+rfc3986==1.5.0
+rich==13.4.2
+rotary-embedding-torch==0.2.3
+rsa==4.9
+s3transfer==0.6.1
+sacrebleu==2.3.1
+sacremoses==0.0.53
+safetensors==0.3.1
+scikit-image==0.21.0
+scikit-learn==1.2.2
+scipy==1.10.1
+segments==2.2.1
+sentencepiece==0.1.99
+sentry-sdk==1.25.1
+setproctitle==1.3.2
+Shapely==1.8.5.post1
+simplejson==3.19.1
+six==1.16.0
+smart-open==6.3.0
+smmap==5.0.0
+soundfile==0.12.1
+soupsieve==2.4.1
+spacy==3.5.3
+spacy-legacy==3.0.12
+spacy-loggers==1.0.4
+srsly==2.4.6
+stempeg==0.2.3
+submitit==1.4.5
+sympy==1.11.1
+tabulate==0.9.0
+tensorboard==2.13.0
+tensorboard-data-server==0.7.1
+termcolor==2.3.0
+thinc==8.1.10
+threadpoolctl==3.1.0
+tifffile==2023.4.12
+timm @ git+https://github.com/huggingface/pytorch-image-models.git@6635bc3f7d06c6a0d0481803b24d6ad0004b61ac
+tinycss2==1.2.1
+tokenizers==0.13.3
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.11.8
+torch-fidelity==0.3.0
+torch-struct @ git+https://github.com/harvardnlp/pytorch-struct@f4e374e894b94a9411fb3d2dfb44201a18e37b26
+torch_geometric @ git+https://github.com/pyg-team/pytorch_geometric.git@cabcd4097442ba60aa1efa11e1619dd9bb8fb527
+torchdata==0.6.1
+torchmetrics==0.11.4
+torchrec-nightly==2023.6.15
+tornado==6.3.2
+tqdm==4.64.1
+# transformers==4.30.0
+treetable==0.2.5
+typer==0.7.0
+typing-inspect==0.9.0
+typing_extensions==4.4.0
+tzdata==2023.3
+Unidecode==1.3.6
+uritemplate==4.1.1
+urllib3==1.26.13
+vector-quantize-pytorch==1.6.19
+visdom==0.2.4
+wandb==0.15.4
+wasabi==1.1.2
+wcwidth==0.2.6
+weasyprint==59.0
+webdataset==0.2.48
+webencodings==0.5.1
+websocket-client==1.6.0
+Werkzeug==2.3.6
+wrapt==1.15.0
+x-clip==0.12.1
+xxhash==3.2.0
+xyzservices==2023.5.0
+yacs==0.1.8
+yarl==1.9.2
+zipp==3.15.0
+zopfli==0.2.2
+transformers==4.31.0
+diffusers==0.18.2
+accelerate==0.21.0
+huggingface-hub==0.16.4

3rds/sources.list

@@ -0,0 +1,11 @@
+#阿里云源
+deb http://mirrors.aliyun.com/ubuntu/ focal main restricted
+deb http://mirrors.aliyun.com/ubuntu/ focal-updates main restricted
+deb http://mirrors.aliyun.com/ubuntu/ focal universe
+deb http://mirrors.aliyun.com/ubuntu/ focal-updates universe
+deb http://mirrors.aliyun.com/ubuntu/ focal multiverse
+deb http://mirrors.aliyun.com/ubuntu/ focal-updates multiverse
+deb http://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse
+deb http://mirrors.aliyun.com/ubuntu/ focal-security main restricted
+deb http://mirrors.aliyun.com/ubuntu/ focal-security universe
+deb http://mirrors.aliyun.com/ubuntu/ focal-security multiverse

3rds/vimrc

@@ -0,0 +1,70 @@
+" All system-wide defaults are set in $VIMRUNTIME/debian.vim and sourced by
+" the call to :runtime you can find below.  If you wish to change any of those
+" settings, you should do it in this file (/etc/vim/vimrc), since debian.vim
+" will be overwritten everytime an upgrade of the vim packages is performed.
+" It is recommended to make changes after sourcing debian.vim since it alters
+" the value of the 'compatible' option.
+
+" This line should not be removed as it ensures that various options are
+" properly set to work with the Vim-related packages available in Debian.
+runtime! debian.vim
+
+" Vim will load $VIMRUNTIME/defaults.vim if the user does not have a vimrc.
+" This happens after /etc/vim/vimrc(.local) are loaded, so it will override
+" any settings in these files.
+" If you don't want that to happen, uncomment the below line to prevent
+" defaults.vim from being loaded.
+" let g:skip_defaults_vim = 1
+
+" Uncomment the next line to make Vim more Vi-compatible
+" NOTE: debian.vim sets 'nocompatible'.  Setting 'compatible' changes numerous
+" options, so any other options should be set AFTER setting 'compatible'.
+"set compatible
+
+" Vim5 and later versions support syntax highlighting. Uncommenting the next
+" line enables syntax highlighting by default.
+if has("syntax")
+  syntax on " Syntax highlighting
+  set showmatch " Shows matching brackets
+  set ruler " Always shows location in file (line#)
+  set smarttab " Autotabs for certain code
+  set shiftwidth=4
+endif
+
+" If using a dark background within the editing area and syntax highlighting
+" turn on this option as well
+"set background=dark
+
+" Uncomment the following to have Vim jump to the last position when
+" reopening a file
+"if has("autocmd")
+"  au BufReadPost * if line("'\"") > 1 && line("'\"") <= line("$") | exe "normal! g'\"" | endif
+"endif
+
+" Uncomment the following to have Vim load indentation rules and plugins
+" according to the detected filetype.
+"if has("autocmd")
+"  filetype plugin indent on
+"endif
+
+" The following are commented out as they cause vim to behave a lot
+" differently from regular Vi. They are highly recommended though.
+"set showcmd		" Show (partial) command in status line.
+"set showmatch		" Show matching brackets.
+"set ignorecase		" Do case insensitive matching
+"set smartcase		" Do smart case matching
+"set incsearch		" Incremental search
+"set autowrite		" Automatically save before commands like :next and :make
+"set hidden		" Hide buffers when they are abandoned
+"set mouse=a		" Enable mouse usage (all modes)
+
+set hlsearch
+if has('mouse')
+  set mouse-=a
+endif
+
+" Source a global configuration file if available
+if filereadable("/etc/vim/vimrc.local")
+  source /etc/vim/vimrc.local
+endif
+

build.sh

@@ -0,0 +1,2 @@
+# docker build -f xtcl.ubuntu2004.Dockerfile --build-arg "https_proxy=http://172.19.56.199:8128" -t xtcl_ubuntu2004:v3.9 .
+docker build -f xtcl.ubuntu2004.Dockerfile -t xtcl_ubuntu2004:v4.6 .

run.sh

@@ -0,0 +1,6 @@
+docker tag xtcl_ubuntu2004:v4.6 iregistry.baidu-int.com/isa/xtcl_ubuntu2004:v4.6
+docker push iregistry.baidu-int.com/isa/xtcl_ubuntu2004:v4.6
+
+# docker run -it --rm xtcl_ubuntu2004:v4.3 java -version
+# docker run -it --rm -d --privileged -v /home/workspace/models/:/home/workspace/models/ -w /home/workspace xtcl_ubuntu2004:v4.3 /bin/bash
+

xtcl.ubuntu2004.Dockerfile

@@ -0,0 +1,76 @@
+FROM ubuntu:20.04
+LABEL maintainer='kunlunxin'
+
+RUN mkdir -p /home/workspace
+WORKDIR /opt/docker
+
+# update system
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Asia/Shanghai
+ENV LANG=zh_CN.UTF-8
+
+# COPY 3rds/sources.list /etc/apt/sources.list
+RUN apt update && apt -y upgrade && apt install -y apt-utils pkg-config nfs-common pciutils build-essential gdb libgmp-dev libz-dev \
+        lsb-release lsb-core apt-transport-https ca-certificates software-properties-common wget curl \
+        vim yasm libgl1-mesa-glx libbz2-dev lzma liblzma-dev libglib2.0-dev libffi-dev libsqlite3-dev \
+	    pigz libgtest-dev automake locales libnl-3-dev libnl-route-3-dev libnl-route-3-200 \
+        libc6-dbg automake tcl-dev gettext libcurl4-openssl-dev ssh-client libsndfile1 ccache \
+		libc6-dev-i386 doxygen swig libssl-dev flex bison clang-format lbzip2 \
+        python3.8 python3.8-dev python3-pip python3.8-venv python3-wheel caffe-cpu libopenmpi-dev libcairo2-dev \
+        python3-cffi python3-brotli libpango-1.0-0 libharfbuzz0b libpangoft2-1.0-0 libgtk2.0-0 \
+        re2c ninja-build libboost-dev libboost-program-options-dev libfftw3-dev libxml2-dev libxmlsec1-dev \
+	libgmock-dev sshpass libopenblas-dev libatlas-base-dev chrpath libtbb-dev libgsl-dev lcov
+
+# install cmake-3.22.3
+# ADD https://github.com/Kitware/CMake/releases/download/v3.22.3/cmake-3.22.3-linux-x86_64.sh .
+COPY 3rds/cmake-3.22.3-linux-x86_64.sh .
+RUN sh ./cmake-3.22.3-linux-x86_64.sh --prefix=/usr --exclude-subdir --skip-license
+
+# install git
+COPY 3rds/v2.34.0.tar.gz .
+RUN tar -zxf v2.34.0.tar.gz && cd git-2.34.0 && make configure && \
+        ./configure --prefix=/usr && make NO_GETTEXT=Yes && make install && git --version 
+
+# install mpi
+RUN wget http://www.mpich.org/static/downloads/3.3.2/mpich-3.3.2.tar.gz && \
+	tar -zxvf mpich-3.3.2.tar.gz && cd mpich-3.3.2 && \
+	./configure  --prefix=/usr/local/mpich-3.3.2 --disable-fortran && \
+	make -j$(nproc) && make install
+
+# install git lfs
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+RUN apt install -y git-lfs && git lfs install 
+
+# install llvm-15
+RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 15 all
+
+# pytorch 2.01
+COPY 3rds/pt201.txt .
+RUN python3 -m venv /home/pt201 && . /home/pt201/bin/activate && \
+	pip3 install --upgrade pip setuptools wheel pyproject-toml && \
+        pip3 install torch-2.0.1+cpu-cp38-cp38-linux_x86_64.whl && \
+        pip3 install torchaudio-2.0.2+cpu-cp38-cp38-linux_x86_64.whl && \
+        pip3 install torchtext-0.15.2+cpu-cp38-cp38-linux_x86_64.whl && \
+        pip3 install torchvision-0.15.2+cpu-cp38-cp38-linux_x86_64.whl && \
+        pip3 install -r pt201.txt && \
+        deactivate
+
+# ?
+RUN wget https://github.com/rui314/mold/releases/download/v1.11.0/mold-1.11.0-x86_64-linux.tar.gz && \
+        tar zxf mold-1.11.0-x86_64-linux.tar.gz && \
+        cp -rf mold-1.11.0-x86_64-linux/bin/* /usr/bin/ && \
+        cp -rf mold-1.11.0-x86_64-linux/lib/* /usr/lib/ && \
+        cp -rf mold-1.11.0-x86_64-linux/libexec/* /usr/libexec/ && \
+        cp -rf mold-1.11.0-x86_64-linux/share/* /usr/share/
+
+# update
+COPY 3rds/xpu_smi.4.30.0.1 /bin/xpu_smi
+COPY 3rds/soft_reset.4.30.0.1 /bin/soft_reset
+
+RUN ln -sf /usr/bin/vim /bin/vi && \
+    ln -sf /usr/bin/python3.8 /usr/bin/python3
+
+RUN locale-gen zh_CN.UTF-8 && \
+    sed -i "/oid_section/a\openssl_conf = default_conf\n [default_conf]\n ssl_conf = ssl_sect\n [ssl_sect]\n system_default = system_default_sect\n [system_default_sect]\n MinProtocol = TLSv1.1\n CipherString = DEFAULT@SECLEVEL=1 " /etc/ssl/openssl.cnf
+
+CMD [". /etc/profile && /bin/bash"]