{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating Ngrams and their Frequency\n",
    "\n",
    "- https://stackoverflow.com/q/32441605/3744499"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.util import ngrams\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I need to write a program in NLTK that breaks a corpus (a large collection of txt files) into unigrams, bigrams, trigrams, fourgrams and fivegrams. I need to write a program in NLTK that breaks a corpus'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = ('I need to write a program in NLTK that breaks a corpus (a large collection of '\n",
    "        'txt files) into unigrams, bigrams, trigrams, fourgrams and fivegrams. '\n",
    "        'I need to write a program in NLTK that breaks a corpus')\n",
    "\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('I', 'need'), ('need', 'to'), ('to', 'write'), ('write', 'a'), ('a', 'program'), ('program', 'in'), ('in', 'NLTK'), ('NLTK', 'that'), ('that', 'breaks'), ('breaks', 'a'), ('a', 'corpus'), ('corpus', '('), ('(', 'a'), ('a', 'large'), ('large', 'collection'), ('collection', 'of'), ('of', 'txt'), ('txt', 'files'), ('files', ')'), (')', 'into'), ('into', 'unigrams'), ('unigrams', ','), (',', 'bigrams'), ('bigrams', ','), (',', 'trigrams'), ('trigrams', ','), (',', 'fourgrams'), ('fourgrams', 'and'), ('and', 'fivegrams'), ('fivegrams', '.'), ('.', 'I'), ('I', 'need'), ('need', 'to'), ('to', 'write'), ('write', 'a'), ('a', 'program'), ('program', 'in'), ('in', 'NLTK'), ('NLTK', 'that'), ('that', 'breaks'), ('breaks', 'a'), ('a', 'corpus')]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Counter({('(', 'a'): 1,\n",
       "         (')', 'into'): 1,\n",
       "         (',', 'bigrams'): 1,\n",
       "         (',', 'fourgrams'): 1,\n",
       "         (',', 'trigrams'): 1,\n",
       "         ('.', 'I'): 1,\n",
       "         ('I', 'need'): 2,\n",
       "         ('NLTK', 'that'): 2,\n",
       "         ('a', 'corpus'): 2,\n",
       "         ('a', 'large'): 1,\n",
       "         ('a', 'program'): 2,\n",
       "         ('and', 'fivegrams'): 1,\n",
       "         ('bigrams', ','): 1,\n",
       "         ('breaks', 'a'): 2,\n",
       "         ('collection', 'of'): 1,\n",
       "         ('corpus', '('): 1,\n",
       "         ('files', ')'): 1,\n",
       "         ('fivegrams', '.'): 1,\n",
       "         ('fourgrams', 'and'): 1,\n",
       "         ('in', 'NLTK'): 2,\n",
       "         ('into', 'unigrams'): 1,\n",
       "         ('large', 'collection'): 1,\n",
       "         ('need', 'to'): 2,\n",
       "         ('of', 'txt'): 1,\n",
       "         ('program', 'in'): 2,\n",
       "         ('that', 'breaks'): 2,\n",
       "         ('to', 'write'): 2,\n",
       "         ('trigrams', ','): 1,\n",
       "         ('txt', 'files'): 1,\n",
       "         ('unigrams', ','): 1,\n",
       "         ('write', 'a'): 2})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token = nltk.word_tokenize(text)\n",
    "bigrams = list(ngrams(token,2))\n",
    "print(bigrams)\n",
    "Counter(bigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I', 'need', 'to'),\n",
       " ('need', 'to', 'write'),\n",
       " ('to', 'write', 'a'),\n",
       " ('write', 'a', 'program'),\n",
       " ('a', 'program', 'in'),\n",
       " ('program', 'in', 'NLTK'),\n",
       " ('in', 'NLTK', 'that'),\n",
       " ('NLTK', 'that', 'breaks'),\n",
       " ('that', 'breaks', 'a'),\n",
       " ('breaks', 'a', 'corpus'),\n",
       " ('a', 'corpus', '('),\n",
       " ('corpus', '(', 'a'),\n",
       " ('(', 'a', 'large'),\n",
       " ('a', 'large', 'collection'),\n",
       " ('large', 'collection', 'of'),\n",
       " ('collection', 'of', 'txt'),\n",
       " ('of', 'txt', 'files'),\n",
       " ('txt', 'files', ')'),\n",
       " ('files', ')', 'into'),\n",
       " (')', 'into', 'unigrams'),\n",
       " ('into', 'unigrams', ','),\n",
       " ('unigrams', ',', 'bigrams'),\n",
       " (',', 'bigrams', ','),\n",
       " ('bigrams', ',', 'trigrams'),\n",
       " (',', 'trigrams', ','),\n",
       " ('trigrams', ',', 'fourgrams'),\n",
       " (',', 'fourgrams', 'and'),\n",
       " ('fourgrams', 'and', 'fivegrams'),\n",
       " ('and', 'fivegrams', '.'),\n",
       " ('fivegrams', '.', 'I'),\n",
       " ('.', 'I', 'need'),\n",
       " ('I', 'need', 'to'),\n",
       " ('need', 'to', 'write'),\n",
       " ('to', 'write', 'a'),\n",
       " ('write', 'a', 'program'),\n",
       " ('a', 'program', 'in'),\n",
       " ('program', 'in', 'NLTK'),\n",
       " ('in', 'NLTK', 'that'),\n",
       " ('NLTK', 'that', 'breaks'),\n",
       " ('that', 'breaks', 'a'),\n",
       " ('breaks', 'a', 'corpus')]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigrams = list(ngrams(token, 3))\n",
    "trigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I', 'need', 'to'),\n",
       " ('need', 'to', 'write'),\n",
       " ('to', 'write', 'a'),\n",
       " ('write', 'a', 'program'),\n",
       " ('a', 'program', 'in'),\n",
       " ('program', 'in', 'NLTK'),\n",
       " ('in', 'NLTK', 'that'),\n",
       " ('NLTK', 'that', 'breaks'),\n",
       " ('that', 'breaks', 'a'),\n",
       " ('breaks', 'a', 'corpus'),\n",
       " ('a', 'corpus', '('),\n",
       " ('corpus', '(', 'a'),\n",
       " ('(', 'a', 'large'),\n",
       " ('a', 'large', 'collection'),\n",
       " ('large', 'collection', 'of'),\n",
       " ('collection', 'of', 'txt'),\n",
       " ('of', 'txt', 'files'),\n",
       " ('txt', 'files', ')'),\n",
       " ('files', ')', 'into'),\n",
       " (')', 'into', 'unigrams'),\n",
       " ('into', 'unigrams', ','),\n",
       " ('unigrams', ',', 'bigrams'),\n",
       " (',', 'bigrams', ','),\n",
       " ('bigrams', ',', 'trigrams'),\n",
       " (',', 'trigrams', ','),\n",
       " ('trigrams', ',', 'fourgrams'),\n",
       " (',', 'fourgrams', 'and'),\n",
       " ('fourgrams', 'and', 'fivegrams'),\n",
       " ('and', 'fivegrams', '.'),\n",
       " ('fivegrams', '.', 'I'),\n",
       " ('.', 'I', 'need'),\n",
       " ('I', 'need', 'to'),\n",
       " ('need', 'to', 'write'),\n",
       " ('to', 'write', 'a'),\n",
       " ('write', 'a', 'program'),\n",
       " ('a', 'program', 'in'),\n",
       " ('program', 'in', 'NLTK'),\n",
       " ('in', 'NLTK', 'that'),\n",
       " ('NLTK', 'that', 'breaks'),\n",
       " ('that', 'breaks', 'a'),\n",
       " ('breaks', 'a', 'corpus'),\n",
       " ('a', 'corpus', None),\n",
       " ('corpus', None, None)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigrams_pad = list(ngrams(token, 3, pad_right=True))\n",
    "trigrams_pad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Built from pure python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import chain\n",
    "\n",
    "def n_grams(tokens, n=1):\n",
    "    \"\"\"Returns an iterator over the n-grams given a list of tokens\"\"\"\n",
    "    shiftToken = lambda i: (el for j,el in enumerate(tokens) if j>=i)\n",
    "    shiftedTokens = (shiftToken(i) for i in range(n))\n",
    "    tupleNGrams = zip(*shiftedTokens)\n",
    "    return tupleNGrams # if join in generator : (\" \".join(i) for i in tupleNGrams)\n",
    "\n",
    "def range_ngrams(tokens, ngramRange=(1,2)):\n",
    "    \"\"\"Returns an itirator over all n-grams for n in range(ngramRange) given a list of tokens.\"\"\"\n",
    "    return chain(*(n_grams(tokens, i) for i in range(*ngramRange)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I', 'need', 'to'),\n",
       " ('need', 'to', 'write'),\n",
       " ('to', 'write', 'a'),\n",
       " ('write', 'a', 'program'),\n",
       " ('a', 'program', 'in'),\n",
       " ('program', 'in', 'NLTK'),\n",
       " ('in', 'NLTK', 'that'),\n",
       " ('NLTK', 'that', 'breaks'),\n",
       " ('that', 'breaks', 'a'),\n",
       " ('breaks', 'a', 'corpus'),\n",
       " ('a', 'corpus', '('),\n",
       " ('corpus', '(', 'a'),\n",
       " ('(', 'a', 'large'),\n",
       " ('a', 'large', 'collection'),\n",
       " ('large', 'collection', 'of'),\n",
       " ('collection', 'of', 'txt'),\n",
       " ('of', 'txt', 'files'),\n",
       " ('txt', 'files', ')'),\n",
       " ('files', ')', 'into'),\n",
       " (')', 'into', 'unigrams'),\n",
       " ('into', 'unigrams', ','),\n",
       " ('unigrams', ',', 'bigrams'),\n",
       " (',', 'bigrams', ','),\n",
       " ('bigrams', ',', 'trigrams'),\n",
       " (',', 'trigrams', ','),\n",
       " ('trigrams', ',', 'fourgrams'),\n",
       " (',', 'fourgrams', 'and'),\n",
       " ('fourgrams', 'and', 'fivegrams'),\n",
       " ('and', 'fivegrams', '.'),\n",
       " ('fivegrams', '.', 'I'),\n",
       " ('.', 'I', 'need'),\n",
       " ('I', 'need', 'to'),\n",
       " ('need', 'to', 'write'),\n",
       " ('to', 'write', 'a'),\n",
       " ('write', 'a', 'program'),\n",
       " ('a', 'program', 'in'),\n",
       " ('program', 'in', 'NLTK'),\n",
       " ('in', 'NLTK', 'that'),\n",
       " ('NLTK', 'that', 'breaks'),\n",
       " ('that', 'breaks', 'a'),\n",
       " ('breaks', 'a', 'corpus')]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(n_grams(token, n=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}