Adjust parameters.

8ffe8c72 · dmt · 9eab159e · 8ffe8c72
Commit 8ffe8c72 authored 4 years ago by dmt
--- a/tests/demo/example.ipynb
+++ b/tests/demo/example.ipynb
@@ -36,10 +36,9 @@
   "source": [
    "settings = cml.get_settings()\n",
    "settings.HIGHEST_LEVEL = 4\n",
-    "settings.KERNEL_BANDWIDTH = 0.01\n",
    "\n",
-    "settings.LEARN_BLOCK_MINIMUM = 2500\n",
-    "settings.SIGMA_ZETA_CUTOFF = 0.2\n",
+    "settings.LEARN_BLOCK_MINIMUM = 5000\n",
+    "settings.SIGMA_ZETA_CUTOFF = 0.01\n",
    "\n",
    "settings.MAX_CATEGORIES = 5\n",
    "settings.MIN_CATEGORY_SIZE = 0.1\n",

 %% Cell type:code id: tags:

 ``` python
 import cml
 ```

 %% Output

    /home/dmt/Project/cml_profiling/venv/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.cluster.birch module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.cluster. Anything that cannot be imported from sklearn.cluster is now part of the private API.
      warnings.warn(message, FutureWarning)
    /home/dmt/Project/cml_profiling/venv/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.kde module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.
      warnings.warn(message, FutureWarning)

 %% Cell type:code id: tags:

 ``` python
 settings = cml.get_settings()
 settings.HIGHEST_LEVEL = 4
-settings.KERNEL_BANDWIDTH = 0.01

-settings.LEARN_BLOCK_MINIMUM = 2500
-settings.SIGMA_ZETA_CUTOFF = 0.2
+settings.LEARN_BLOCK_MINIMUM = 5000
+settings.SIGMA_ZETA_CUTOFF = 0.01

 settings.MAX_CATEGORIES = 5
 settings.MIN_CATEGORY_SIZE = 0.1

 settings.MIN_BUILD_MODELS = 2
 settings.MAX_FEATURES = 60
 settings.MAX_FILTER_X = 50
 settings.MAX_FILTER_Y = 15000
 settings.MAX_MODELS_REDUCTION = True

 settings.MIN_TEST_ACCURACY = 0.8
 settings.RELIABILITY_SAMPLE = 0.3
 settings.MIN_RELIABILITY = 0.8
 settings.REDUCE_MODEL_REDUNDANCY = False

 settings.DECONST_STRATEGY = "integrative"
 settings.DECONST_MODE = "minimal"
 settings.DECONST_MAX_DISTANCE_T = 1.0
 settings.DECONST_FULL_TOLERANCE = 0.1
 settings.FORCE_TIME_EXPANSION = False
 settings.ALLOW_WEAK_RELIABILITY = True
 ```

 %% Cell type:code id: tags:

 ``` python
 import os
 import pandas as pd

 PATH = os.path.join(os.path.expanduser("~"), "final_test.csv")

 def generate_blocks():
    df = pd.read_table(PATH, index_col=False,  sep=" ",
                       names=[i for i in range(1,353)]+["T", "Sigma", "Z"])
    df["Z"] = ""
    df["Sigma"] = ""

    for from_, to in [
        (0, 83), (0, 41), (42, 83),
        (84, 167), (84, 125), (126, 167),
        (168, 249), (168, 208), (209, 249),
        (250, 281), (250, 265), (266, 281),
        (282, 313), (282, 297), (298, 313),
        (314, 345), (314, 329), (330, 345)
    ]:
        for start in range(0, 10000, 10000):
            bl = df.iloc[start:start + 10000, from_:to]
            bl.columns = [f"0.0.{i}" for i in bl.columns]
            bl["T"], bl["Sigma"], bl["Z"] = df["T"], df["Sigma"], df["Z"]
            yield bl
 ```

 %% Cell type:code id: tags:

 ``` python
 from sklearn.cluster import KMeans
 from sklearn.cluster import AgglomerativeClustering

 kmeans = KMeans(precompute_distances=False, algorithm="elkan", random_state=1)
 agglo = AgglomerativeClustering()
 algorithms = [("Kme", kmeans), ("Agg", agglo)]

 constructor = cml.construction("conceptual", algorithms)
 ```

 %% Cell type:code id: tags:

 ``` python
 from sklearn.feature_selection import VarianceThreshold, SelectFromModel
 from sklearn.ensemble import ExtraTreesClassifier

 variance = VarianceThreshold(2000)
 forest = ExtraTreesClassifier(n_estimators=250, random_state=1)
 embedded = SelectFromModel(forest)

 selector = cml.feature_selection(filter_method=variance, embedded_method=embedded)
 ```

 %% Cell type:code id: tags:

 ``` python
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.svm import SVC
 from sklearn.neighbors import KNeighborsClassifier
 from sklearn.naive_bayes import GaussianNB

 random_forest = RandomForestClassifier(n_estimators=2)
 svc = SVC(gamma="auto", C=1)
 neighbor = KNeighborsClassifier(n_neighbors=3)
 gaus = GaussianNB()
 algos = [("Gau", gaus),("Nei", neighbor)]

 reconstructor = cml.reconstruction("conceptual", algos)
 ```

 %% Cell type:code id: tags:

 ``` python
 deconstructor = cml.deconstruction("conceptual")
 ```

 %% Cell type:code id: tags:

 ``` python
 dbs = []
 components = (constructor, selector, reconstructor, deconstructor)
 with cml.knowledge_searcher(*components, n_procs=4) as s:
    for i, block in enumerate(generate_blocks()):
        db, halde = s.search(block)
        dbs.append(db)
 ```

 %% Output

    ==========================================BLOCKPROCESSING===========================================
    Rows                10000
    Columns             86

 %% Cell type:code id: tags:

 ``` python
 for level in db.database:
    print("{:*^100}".format(level.level))
    for m in level.knowledge:
        print(m.uid, m.min_timestamp, m.max_timestamp, m.subjects, m.aim, m.origin, m.subject)
 ```

 %% Output

    *************************************************0**************************************************
    *************************************************1**************************************************
    C.1.68 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Kme02',) (4,) ('Nei',)
    C.1.99 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg03',) (0, 5) ('Nei',)
    C.1.118 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Kme02',) (7,) ('Nei',)
    C.1.149 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg02',) (1, 2, 3, 6, 8) ('Nei',)
    C.1.219 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg03',) (13,) ('Gau',)
    C.1.247 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg02',) (12, 14) ('Nei',)
    C.1.249 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg02',) (15,) ('Nei',)
    C.1.264 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg02',) (11, 12, 15) ('Nei',)
    C.1.268 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg02',) (16,) ('Nei',)
    C.1.282 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg02',) (17,) ('Nei',)
    C.1.297 1473880956 1522686863 ('Gau', 'Nei') ('C.1.Agg02',) (17, 14) ('Nei',)
    *************************************************2**************************************************
    C.2.38 1473880956 1522686863 ('Gau', 'Nei') ('C.2.Agg03',) ('C.1.118', 'C.1.68') ('Nei',)
    C.2.79 1473880956 1522686863 ('Gau', 'Nei') ('C.2.Agg05',) ('C.1.219', 'C.1.68') ('Nei',)
    C.2.88 1473880956 1522686863 ('Gau', 'Nei') ('C.2.Agg03',) ('C.1.268', 'C.1.68') ('Nei',)
    *************************************************3**************************************************
    C.3.31 1473880956 1522686863 ('Gau', 'Nei') ('C.3.Agg05',) ('C.2.79', 'C.2.38') ('Nei',)
    C.3.40 1473880956 1522686863 ('Gau', 'Nei') ('C.3.Agg03',) ('C.2.88', 'C.2.38') ('Nei',)
    *************************************************4**************************************************
    C.4.11 1473880956 1522686863 ('Gau', 'Nei') ('C.4.Agg02',) ('C.3.40', 'C.3.31') ('Nei',)

 %% Cell type:code id: tags:

 ``` python
 for i in str(dbs[-1]).split("\n"):
    print(i)
 ```