feature_extraction__classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9eb0014d",
   "metadata": {},
   "source": [
    "# Baseline text classification experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d979d6f8",
   "metadata": {},
   "source": [
    "This notebook builds and tests the baseline \"human\"/\"machine\" text classifier proposed in [TBD-citation](link to publication) by first extracting the features from the training datasets and then testing multiple classifiers on the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f963291",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5606272",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "021f41dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a14d7393",
   "metadata": {},
   "source": [
    "## Feature Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26b42680",
   "metadata": {},
   "source": [
    "### Load training datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c45038d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpora = []\n",
    "labels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26e8ad7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data_path = os.path.join(data_path, 'training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2367ab1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt2xl_df = pd.read_json(os.path.join(training_data_path, 'GPT2-xl-1542M.train.filtered.jsonl'), lines = True)\n",
    "gpt2xl_k40_df = pd.read_json(os.path.join(training_data_path, 'GPT2-xl-1542M-k40.train.filtered.jsonl'), lines = True)\n",
    "webtext_df = pd.read_json(os.path.join(training_data_path, 'webtext.train.filtered.jsonl'), lines = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7ae859b",
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = 100000"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8e431d8",
   "metadata": {},
   "source": [
    "Training set 1: 200k samples, half drawn from WebText and the rest from GPT-2 (random) generations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1879b0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpora.append(webtext_df.sample(n=samples, random_state=seed)['text'].to_list() + gpt2xl_df.sample(n=samples, random_state=seed)['text'].to_list())\n",
    "labels.append([0 for _ in range(samples)] + [1 for _ in range(samples)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f42ba1b",
   "metadata": {},
   "source": [
    "Training set 2: 200k samples, half drawn from WebText and the rest from GPT-2 (k40) generations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "106269f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpora.append(webtext_df.sample(n=samples, random_state=seed)['text'].to_list() + gpt2xl_k40_df.sample(n=samples, random_state=seed)['text'].to_list())\n",
    "labels.append([0 for _ in range(samples)] + [1 for _ in range(samples)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cddcd2db",
   "metadata": {},
   "source": [
    "Training set 3: 400k samples,  half drawn from WebText, and the rest equally sampled from GPT-2 (random)\n",
    "and GPT-2 (k40) generations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a9c4200",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpora.append(webtext_df.sample(n=2*samples, random_state=seed)['text'].to_list() + gpt2xl_df.sample(n=samples, random_state=seed)['text'].to_list() + gpt2xl_k40_df.sample(n=samples, random_state=seed)['text'].to_list())\n",
    "labels.append([0 for _ in range(2*samples)] + [1 for _ in range(2*samples)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5509da0b",
   "metadata": {},
   "source": [
    "### Build vocabularies"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01b790b2",
   "metadata": {},
   "source": [
    "To prevent words exclusive to one or the other dataset to influence classifiers downstream, shared vocabularies that act as \"whitelists\" to retain tokens from the training sets are first computed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e1997c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import preprocessing as pp\n",
    "from sklearn.pipeline import make_pipeline\n",
    "\n",
    "def tokenize_corpus(corpus, whitelist=None):\n",
    "    tokenized_corpus = make_pipeline(\n",
    "        pp.WordTokenizer(), \n",
    "        pp.WordsFilter(drop_symbols=False, drop_digits=True, whitelist=whitelist)\n",
    "    ).fit_transform(corpus)\n",
    "    return tokenized_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e57bf5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_corpora = [tokenize_corpus(corpus) for corpus in corpora]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea3612b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_shared = []\n",
    "for i, tc in enumerate(tokenized_corpora):\n",
    "    vocab_human = pp.get_vocabulary(tc[:int(len(tc)/2)])\n",
    "    vocab_machine = pp.get_vocabulary(tc[int(len(tc)/2):])\n",
    "    vocab_shared.append(vocab_human.intersection(vocab_machine))\n",
    "    print(f'Training set ({i+1}) vocabulary sizes:\\n  \"human\":\\t{len(vocab_human)}\\n  \"machine:\\t{len(vocab_machine)}\\n  \"shared\":\\t{len(vocab_shared[i])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b466cf86",
   "metadata": {},
   "source": [
    "### Tokenize and filter datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e96d2f3",
   "metadata": {},
   "source": [
    "Training datasets are tokenized and filtered again using the computed shared vocabolaries as whitelists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fac9100",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_corpora = [tokenize_corpus(corpus, whitelist=vocab) for (corpus, vocab) in zip(corpora, vocab_shared)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc0d20ce",
   "metadata": {},
   "source": [
    "Test datasets are also preprocess using the same approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f1ebb52",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_path = os.path.join(data_path, 'test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38f149f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = []\n",
    "test_labels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fedebb5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_files = sorted([f for f in os.listdir(test_data_path) if os.path.isfile(os.path.join(test_data_path, f)) and f[0]!=\".\"])\n",
    "for test_data_file in test_data_files:\n",
    "    df = pd.read_json(os.path.join(test_data_path, test_data_file), lines = True)\n",
    "    test_data.append(df['text'].to_list())\n",
    "    if 'human' in test_data_file:\n",
    "        test_labels.append([0 for _ in range(len(df))])\n",
    "    else:\n",
    "        test_labels.append([1 for _ in range(len(df))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09033c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_test_data = [tokenize_corpus(data) for data in test_data]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c10f577",
   "metadata": {},
   "source": [
    "### POS tagging with Spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e7989e6",
   "metadata": {},
   "source": [
    "Extract POS tags using [Spacy](https://spacy.io/) (_en_core_web_sm_ model for English) from the corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9db629af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\", enable=['tok2vec', 'tagger'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "559c5d7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import sys\n",
    "\n",
    "fmt = \"  Progress: {:>3}% estimated {:>3}s remaining\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5eb2a30e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pos_tags(datasets):\n",
    "    tagged_datasets = []\n",
    "    for i, dataset in enumerate(datasets):\n",
    "        start = time.perf_counter()\n",
    "        print(f'Extracting POS tags from dataset {i+1} of {len(datasets)}')\n",
    "        size = len(dataset)\n",
    "        sys.stdout.write(f\" processing text 1/{size}\")\n",
    "        tagged_dataset = []\n",
    "        j = 0\n",
    "        for doc in nlp.pipe(dataset):\n",
    "            tokens = nlp(doc)\n",
    "            tagged_dataset.append([tk.tag_ for tk in tokens if not tk.is_stop])\n",
    "            stop = time.perf_counter()\n",
    "            remaining = round((stop - start) * (size / (j+1) - 1))\n",
    "            sys.stdout.write(f\"\\r processing text {j+1}/{size} - {fmt.format(100 * (j+1) // size, remaining)}\")\n",
    "            j += 1\n",
    "        tagged_datasets.append(tagged_dataset)\n",
    "        print('\\n')\n",
    "    return tagged_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68478432",
   "metadata": {},
   "outputs": [],
   "source": [
    "tagged_corpora = get_pos_tags(corpora)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb6c7216",
   "metadata": {},
   "source": [
    "Extract POS tags from test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aecc8b66",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tagged_test_data = get_pos_tags(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b61e28e1",
   "metadata": {},
   "source": [
    "## TF-IDF with N-Grams"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca439203",
   "metadata": {},
   "source": [
    "Build weighted document-term matrices to vectorize textual data from the training data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77ada06a",
   "metadata": {},
   "source": [
    "### Model fitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25df95d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e32d3c57",
   "metadata": {},
   "outputs": [],
   "source": [
    "def identity(x):\n",
    "    return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c6688c6",
   "metadata": {},
   "source": [
    "Representation for N-grams of up to 3 words are fitted on the tokenized training corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4366cb00",
   "metadata": {},
   "outputs": [],
   "source": [
    "words_pipelines = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a29d2169",
   "metadata": {},
   "outputs": [],
   "source": [
    "for X in tokenized_corpora:\n",
    "    pipeline = make_pipeline(\n",
    "        TfidfVectorizer(\n",
    "            ngram_range=(1,3), \n",
    "            max_features=1000000, \n",
    "            sublinear_tf=True,\n",
    "            min_df=3,\n",
    "            tokenizer=identity, \n",
    "            preprocessor=identity),\n",
    "        TruncatedSVD(n_components=500)\n",
    "    )\n",
    "    pipeline.fit(X)\n",
    "    words_pipelines.append(pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5761b9da",
   "metadata": {},
   "source": [
    "Representation for N-grams of 3 to 5 POS tags are fitted on the training corpus preprocessed using Spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4477108",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_pipelines = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "813da6f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "for X in tagged_corpora:\n",
    "    pipeline = make_pipeline(\n",
    "        TfidfVectorizer(\n",
    "            ngram_range=(3,5), \n",
    "            max_features=1000000, \n",
    "            sublinear_tf=True,\n",
    "            tokenizer=identity, \n",
    "            preprocessor=identity),\n",
    "        TruncatedSVD(n_components=300)\n",
    "    )\n",
    "    pipeline.fit(X)\n",
    "    tags_pipelines.append(pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a7041dd",
   "metadata": {},
   "source": [
    "### Dataset vectorization"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbac1f02",
   "metadata": {},
   "source": [
    "The fitted TF-IDF model is used to vectorize training and testing datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad8ec916",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_corpora = [pipe.transform(ds) for (pipe, ds) in zip(words_pipelines, tokenized_corpora)]\n",
    "embed_tagged_corpora = [pipe.transform(ds) for (pipe, ds) in zip(tags_pipelines, tagged_corpora)]\n",
    "embed_combined_corpora = [np.concatenate([ec, etc], axis=1) for (ec, etc) in zip(embed_corpora, embed_tagged_corpora)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2978370d",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_test_data = [[pipe.transform(ds) for ds in tokenized_test_data] for pipe in words_pipelines]\n",
    "embed_tagged_test_data = [[pipe.transform(ds) for ds in tagged_test_data] for pipe in tags_pipelines]\n",
    "embed_combined_test_data = [[np.concatenate([etd, ettd], axis=1) for (etd, ettd) in zip(etds, ettds)] for (etds, ettds) in zip(embed_test_data, embed_tagged_test_data)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b9c1c68",
   "metadata": {},
   "source": [
    "## Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ef88fcd",
   "metadata": {},
   "source": [
    "The classification task is addressed using [XGBoost](https://xgboost.readthedocs.io/en/latest/index.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da69547d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "import xgboost as xgb\n",
    "import numpy as np\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dc4d41e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# classifiers trained on words-only TF-IDF embeddings\n",
    "words_classifiers = []\n",
    "# classifiers trained on POS tags-only TF-IDF embeddings\n",
    "tags_classifiers = []\n",
    "# classifiers trained on words and POS tags TF-IDF combined embeddings\n",
    "combined_classifiers = []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae2316d9",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31f9a7ab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for (embed_corpus, embed_tagged_corpus, embed_combined_corpus, y) in zip(embed_corpora, embed_tagged_corpora, embed_combined_corpora, labels):\n",
    "    for (X, classifiers) in zip([embed_corpus, embed_tagged_corpus, embed_combined_corpus], [words_classifiers, tags_classifiers, combined_classifiers]):\n",
    "        clf = GridSearchCV(\n",
    "            xgb.XGBClassifier(n_jobs=multiprocessing.cpu_count() // 2, tree_method=\"hist\"),\n",
    "            {\"max_depth\": [3, 4, 5], \"n_estimators\": [500, 1000, 1500]},\n",
    "            #{\"max_depth\": [3], \"n_estimators\": [1500]},  # best recorded settings\n",
    "            verbose=1,\n",
    "            n_jobs=2)\n",
    "        clf.fit(X, y)\n",
    "        classifiers.append(clf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58494b84",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99d98f96",
   "metadata": {},
   "source": [
    "Compute prediction accuracy for each test datasets (binary-classification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb695022",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af458442",
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictions of classifiers trained on words-only TF-IDF embeddings\n",
    "words_clf_predictions = []\n",
    "# predictions of classifiers trained on POS tags-only TF-IDF embeddings\n",
    "tags_clf_predictions = []\n",
    "# predictions of classifiers trained on words and POS tags TF-IDF combined embeddings\n",
    "combined_clf_predictions = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86483eea",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(embed_test_data)):\n",
    "    w_preds = []\n",
    "    t_preds = []\n",
    "    wt_preds = []\n",
    "    for (embed_test_set, embed_tag_test_set, embed_combined_test_set) in zip(\n",
    "        embed_test_data[i], embed_tagged_test_data[i], embed_combined_test_data[i]):\n",
    "        for (X, clf, preds) in zip([embed_test_set, embed_tag_test_set, embed_combined_test_set],\n",
    "                                   [words_classifiers[i], tags_classifiers[i], combined_classifiers[i]],\n",
    "                                   [w_preds, t_preds, wt_preds]):\n",
    "            preds.append([round(y_pred) for y_pred in clf.best_estimator_.predict(X)])\n",
    "    for preds, i_preds in zip([words_clf_predictions, tags_clf_predictions, combined_clf_predictions], [w_preds, t_preds, wt_preds]):\n",
    "        preds.append(i_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d502cef",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = []\n",
    "for i, test_data_file in enumerate(test_data_files):\n",
    "    results_line = [\".\".join(test_data_file.split(\".test\")[0:1])]\n",
    "    if \"machine\" in test_data_file:\n",
    "        results_line.append(\"machine\")\n",
    "    else:\n",
    "        results_line.append(\"human\")\n",
    "    results_line.append(len(test_labels[i]))\n",
    "    for (words_ds_pred, tags_ds_pred, combined_ds_pred) in zip(words_clf_predictions, tags_clf_predictions, combined_clf_predictions):\n",
    "        for clf_pred in [words_ds_pred, tags_ds_pred, combined_ds_pred]:\n",
    "            results_line.append(accuracy_score(test_labels[i], clf_pred[i]))\n",
    "    results.append(results_line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb12fbe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\"dataset\", \"source\", \"size\"]\n",
    "for i in range(len(words_clf_predictions)):\n",
    "    cols += [f\"(ds-{i+1}) words\", f\"(ds-{i+1}) tags\", f\"(ds-{i+1}) words+tags\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b64280bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.DataFrame(results, columns=cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19ec2b15",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3050f8a1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:gtd-env]",
   "language": "python",
   "name": "conda-env-gtd-env-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}