diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
index adae95b..822e71e 100644
--- a/.github/workflows/python-app.yml
+++ b/.github/workflows/python-app.yml
@@ -26,9 +26,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        chmod +x install.sh
         pip install -e .
-        ./install.sh
         pip install flake8 pytest
     - name: Test with pytest
       run: |
diff --git a/README.md b/README.md
index e387c5f..578c617 100644
--- a/README.md
+++ b/README.md
@@ -21,21 +21,7 @@ A comprehensive toolkit for applying active learning techniques to natural langu
 
 ## 🔧 Installation
 
-1. Clone the repository:
-   ```bash
-   git clone https://github.com/Aktsvigun/atgen.git
-   cd atgen
-   ```
-
-2. Run the installation script:
-   ```bash
-   bash install.sh
-   ```
-
-   This will install:
-   - All required Python packages
-   - External metrics (submodlib, AlignScore)
-   - Required NLP resources
+`pip install atgen`
 
 ## 🚀 Usage
 
@@ -115,4 +101,4 @@ If you use this toolkit in your research, please cite:
   url = {https://github.com/Aktsvigun/atgen},
   year = {2025},
 }
-``` 
\ No newline at end of file
+```
diff --git a/install.sh b/install.sh
deleted file mode 100755
index 2a30fa3..0000000
--- a/install.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# The PyPI version submodlib seems to be broken, so
-# one has to resort to manual installation.
-mkdir external_metrics
-cd external_metrics
-echo "Installing submodlib"
-git clone https://github.com/decile-team/submodlib.git
-cd submodlib
-# Cannot run `pip install -r requirements.txt` due to versions mismatch
-pip install sphinxcontrib-bibtex pybind11>=2.6.0 scikit-learn scipy
-pip install . --no-deps
-cd ..
-rm -rf submodlib
-
-# Install external metrics
-echo "Installing AlignScore..."
-git clone https://github.com/yuh-zha/AlignScore
-cd AlignScore
-
-# Fix the classmethod bug in inference.py
-echo "Patching AlignScore code to fix classmethod bug..."
-sed -i 's/BERTAlignModel(model=model).load_from_checkpoint(checkpoint_path=ckpt_path, strict=False)/BERTAlignModel.load_from_checkpoint(checkpoint_path=ckpt_path, model=model, strict=False)/g' src/alignscore/inference.py
-
-pip install . --no-deps
-mkdir model
-wget https://huggingface.co/yzha/AlignScore/resolve/main/AlignScore-base.ckpt -P model
-cd ../..
-
-echo "Installing the package..."
-pip install -e .
-
-echo "Downloading NLP packages data..."
-python -m spacy download en_core_web_sm
-python3 -c "import nltk ; nltk.download('punkt'); nltk.download('punkt_tab')"
-
-echo "Done!"
diff --git a/pyproject.toml b/pyproject.toml
index 125ac23..bcfe10d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "atgen"
-version = "0.0.0"
+version = "0.1.0"
 authors = [
   { name = "List of contributors: https://github.com/Aktsvigun/atgen/graphs/contributors", email = "artemshelmanov@gmail.com" },
 ]
@@ -12,7 +12,7 @@ description = "Toolkit for Active Learning in Generative Tasks"
 readme = "README.md"
 
 keywords = ["NLP", "deep learning", "transformer", "pytorch", "peft", "inference", "active learning"]
-license = {file = "LICENSE.md"}
+license-files = ["LICENSE.md"]
 
 requires-python = ">=3.10"
 dynamic = ["dependencies"]
diff --git a/requirements.txt b/requirements.txt
index 11b1ce5..216563c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+alignscore-SpeedOfMagic
 accelerate==1.5.2
 anthropic==0.49.0
 benepar==0.2.0
diff --git a/src/atgen/metrics/metrics.py b/src/atgen/metrics/metrics.py
index a22ec76..52f9504 100644
--- a/src/atgen/metrics/metrics.py
+++ b/src/atgen/metrics/metrics.py
@@ -62,7 +62,7 @@
     # Going up 3 levels from metrics.py: src/atgen/metrics -> repository root
     os.path.join(
         Path(__file__).parents[3],
-        "external_metrics/AlignScore/model/AlignScore-base.ckpt",
+        "cache/AlignScore-base.ckpt",
     ),
 )
 
diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py
index 728f1ef..a3747a1 100644
--- a/src/atgen/run_scripts/run_active_learning.py
+++ b/src/atgen/run_scripts/run_active_learning.py
@@ -32,6 +32,7 @@ def run_active_learning(config, workdir: Union[str, Path]):
         maybe_get_few_shot_examples,
         get_output_column_name_for_phase,
     )
+    from atgen.utils.installers import install_spacy, install_nltk
     from atgen.utils.load_model_tokenizer import load_model_tokenizer
     from atgen.utils.prepare_model_for_training import prepare_model_for_training
     from atgen.utils.training_utils import get_trainer
@@ -50,6 +51,10 @@ def run_active_learning(config, workdir: Union[str, Path]):
         check_performance_against_requirements,
     )
 
+    # TODO Figure out how to stop downloading it every time
+    install_spacy()
+    install_nltk()
+
     seed = config.seed
     cache_dir = config.cache_dir
     input_column_name = config.data.input_column_name
diff --git a/src/atgen/utils/installers.py b/src/atgen/utils/installers.py
new file mode 100644
index 0000000..b540fb9
--- /dev/null
+++ b/src/atgen/utils/installers.py
@@ -0,0 +1,10 @@
+import subprocess
+
+
+def install_spacy():
+    subprocess.run("python -m spacy download en_core_web_sm", shell=True)
+
+def install_nltk():
+    import nltk
+    nltk.download('punkt')
+    nltk.download('punkt_tab')