RAG with Azure Document Intelligence and Azure OpenAI/RAG with Azure Document Intelligence and Azure OpenAI gpt-4o mini (Document analysis).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c006b536",
   "metadata": {},
   "source": [
    "# RAG with Azure Document Intelligence and Azure OpenAI gpt-4o mini (document analysis)\n",
    "\n",
    "## Architecture\n",
    "<img src=\"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/media/rag/azure-rag-processing.png\" width=800>\n",
    "\n",
    "\n",
    "## Note\n",
    "Document Intelligence public preview releases provide early access to features that are in active development. Features, approaches, and processes may change, prior to General Availability (GA), based on user feedback.<br>\n",
    "\n",
    "The public preview version of Document Intelligence client libraries default to REST API version 2024-07-31-preview.\n",
    "Public preview version 2024-07-31-preview is currently only available in the following Azure regions. Note that the custom generative (document field extraction) model in AI Studio is only available in North Central US region:\n",
    "- East US\n",
    "- West US2\n",
    "- West Europe\n",
    "- North Central US\n",
    "\n",
    "## Documentation\n",
    "\n",
    "https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept-layout?view=doc-intel-4.0.0&tabs=sample-code\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5522de9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gradio as gr\n",
    "import json\n",
    "import langchain\n",
    "import matplotlib.pyplot as plt\n",
    "import openai\n",
    "import os\n",
    "import pytz\n",
    "import requests\n",
    "import sys\n",
    "import time\n",
    "\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from dotenv import load_dotenv\n",
    "from datetime import datetime\n",
    "from langchain import hub\n",
    "from langchain.schema import StrOutputParser\n",
    "from langchain.schema.runnable import RunnableMap\n",
    "from langchain.schema.runnable import RunnablePassthrough\n",
    "from langchain.text_splitter import MarkdownHeaderTextSplitter\n",
    "from langchain.vectorstores.azuresearch import AzureSearch\n",
    "from langchain_community.document_loaders import AzureAIDocumentIntelligenceLoader\n",
    "from langchain_openai import AzureChatOpenAI\n",
    "from langchain_openai import AzureOpenAIEmbeddings\n",
    "from operator import itemgetter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "80632fc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.10.11 (main, May 16 2023, 00:28:57) [GCC 11.2.0]\n",
      "OpenAI version: 1.43.0\n",
      "Langchain version: 0.2.11\n"
     ]
    }
   ],
   "source": [
    "print(f\"Python version: {sys.version}\")\n",
    "print(f\"OpenAI version: {openai.__version__}\")\n",
    "print(f\"Langchain version: {langchain.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "33da29fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Local time in Europe/Amsterdam: 06-Sep-2024 10:46:25\n"
     ]
    }
   ],
   "source": [
    "# Local timezone datetime\n",
    "local_tz = pytz.timezone(requests.get(\"https://ipinfo.io\").json()[\"timezone\"])\n",
    "now = datetime.now(local_tz).strftime(\"%d-%b-%Y %H:%M:%S\")\n",
    "print(f\"Local time in {local_tz}: {now}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9fb9cc9",
   "metadata": {},
   "source": [
    "## 1. Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7e0e0cec",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv(\"azure.env\")\n",
    "\n",
    "# Azure Document Intelligence\n",
    "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT: str = os.getenv(\"AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT\")\n",
    "AZURE_DOCUMENT_INTELLIGENCE_KEY: str = os.getenv(\"AZURE_DOCUMENT_INTELLIGENCE_KEY\")\n",
    "\n",
    "# Azure OpenAI   \n",
    "AOAI_API_ENDPOINT: str  = os.getenv(\"AZURE_OPENAI_API_ENDPOINT\")\n",
    "AOAI_API_KEY: str  = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
    "\n",
    "# Azure AI Search\n",
    "AZURE_AISEARCH_ENDPOINT: str = os.getenv(\"AZURE_SEARCH_ENDPOINT\")\n",
    "AZURE_AISEARCH_KEY: str = os.getenv(\"AZURE_SEARCH_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9b1cf7af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Azure OpenAI embeddings model\n",
    "AOAI_EMBEDDINGS_MODEL: str = \"text-embedding-ada-002\"\n",
    "\n",
    "# GPT model\n",
    "AOAI_DEPLOYMENT_NAME: str = \"gpt-4o-mini\"\n",
    "# API Version\n",
    "AOAI_API_VERSION: str = \"2024-07-01-preview\"\n",
    "# Type\n",
    "AOAI_API_TYPE: str = \"azure\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "826297cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Azure AI Search Index to create\n",
    "azure_aisearch_index_name: str = \"index_\" + datetime.now(local_tz).strftime(\"%d%b%Y_%H%M%S\").lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2db69e37",
   "metadata": {},
   "source": [
    "## 2. File to analyse with Azure Document Intelligence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ed2e4d1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "DOCS_DIR = \"documents\"\n",
    "\n",
    "os.makedirs(DOCS_DIR, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a6ba706a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-09-06 08:46:25--  https://github.com/retkowsky/Azure-OpenAI-demos/raw/main/documents/phi3.pdf\n",
      "Resolving github.com (github.com)... 140.82.121.4\n",
      "Connecting to github.com (github.com)|140.82.121.4|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://raw.githubusercontent.com/retkowsky/Azure-OpenAI-demos/main/documents/phi3.pdf [following]\n",
      "--2024-09-06 08:46:25--  https://raw.githubusercontent.com/retkowsky/Azure-OpenAI-demos/main/documents/phi3.pdf\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.110.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 3854167 (3.7M) [application/octet-stream]\n",
      "Saving to: ‘documents/phi3.pdf’\n",
      "\n",
      "documents/phi3.pdf  100%[===================>]   3.67M  --.-KB/s    in 0.01s   \n",
      "\n",
      "2024-09-06 08:46:25 (247 MB/s) - ‘documents/phi3.pdf’ saved [3854167/3854167]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "document_file = os.path.join(DOCS_DIR, \"phi3.pdf\")\n",
    "\n",
    "!wget https://github.com/retkowsky/Azure-OpenAI-demos/raw/main/documents/phi3.pdf -O $document_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "161817cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rwxrwxrwx 1 root root 3.7M Sep  6 08:46 documents/phi3.pdf\r\n"
     ]
    }
   ],
   "source": [
    "!ls $document_file -lh"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e1ba1c1",
   "metadata": {},
   "source": [
    "### Azure Document Intelligence layout example\n",
    "\n",
    "<img src=\"https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/media/rag/markdown-text-output.png?view=doc-intel-4.0.0\">\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db058b9d",
   "metadata": {},
   "source": [
    "- **Simplified processing.** You can parse different document types, such as digital and scanned PDFs, images, office files (docx, xlsx, pptx), and HTML, with just a single API call.\n",
    "\n",
    "- **Scalability and AI quality.** The Layout model is highly scalable in Optical Character Recognition (OCR), table extraction, and document structure analysis. It supports 309 printed and 12 handwritten languages, further ensuring high-quality results driven by AI capabilities.\n",
    "\n",
    "- **Large language model (LLM) compatibility.** The Layout model Markdown formatted output is LLM friendly and facilitates seamless integration into your workflows. You can turn any table in a document into Markdown format and avoid extensive effort parsing the documents for greater LLM understanding."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2f8eb32",
   "metadata": {},
   "source": [
    "The task of recognizing small text from large-size documents, like engineering drawings, is a challenge. Often the text is mixed with other graphical elements and has varying fonts, sizes, and orientations. Moreover, the text can be broken into separate parts or connected with other symbols. Document Intelligence now supports extracting content from these types of documents with the **ocr.highResolution** capability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cb0620f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing the document with Azure Document Intelligence...\n",
      "\n",
      "Done\n",
      "Elapsed time = 0 minutes and 14 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "loader = AzureAIDocumentIntelligenceLoader(\n",
    "    file_path=document_file,  # File to process\n",
    "    api_key=AZURE_DOCUMENT_INTELLIGENCE_KEY, # Key\n",
    "    api_endpoint=AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT,  # Endpoint\n",
    "    api_model=\"prebuilt-layout\",  # Prebuilt layout of Azure Doc Intelligence\n",
    "    analysis_features=[\"ocrHighResolution\"],  # To get OCR High resolution\n",
    ")\n",
    "\n",
    "# Loading the document\n",
    "print(\"Processing the document with Azure Document Intelligence...\")\n",
    "documents = loader.load()\n",
    "\n",
    "elapsed = time.time() - start\n",
    "minutes, seconds = divmod(elapsed, 60)\n",
    "print(\"\\nDone\")\n",
    "print(f\"Elapsed time = {minutes:.0f} minutes and {seconds:.0f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "105a926c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone\\n===\\n\\nMicrosoft\\n\\n\\n# Abstract\\n\\nWe introduce phi-3-mini, a 3.8 billion parameter language model trained on 3.3 trillion tokens, whose overall performance, as measured by both academic benchmarks and internal testing, rivals that of models such as Mixtral 8x7B and GPT-3.5 (e.g., phi-3-mini achieves 69% on MMLU and 8.38 on MT-bench), despite being small enough to be deployed on a phone. The innovation lies entirely in our dataset for training, a scaled-up version of the one used for phi-2, composed of heavily filtered publicly available web data and synthetic data. The model is also further aligned for robustness, safety, and chat format. We also provide some initial parameter-scaling results with a 7B and 14B models trained for 4.8T tokens, called phi-3-small and phi-3-medium, both significantly more capable than phi-3-mini (e.g., respectively 75% and 78% on MMLU, and 8.7 and 8.9 on MT-bench). Moreover, we also introduce phi-3-vision, a 4.2 billion parameter model based on phi-3-mini with strong reasoning capabilities for image and text prompts.\\n\\n\\n# Introduction 1\\n\\nThe striking progress of AI in the last few years can be largely attributed to major efforts through- out the world towards scaling-up to ever-larger models and datasets. Large Language Models (LLMs) have steadily increased in size from a mere billion parameters just five years ago (GPT-2 had 1.5 bil- lion parameters [RWC+19]) to trillion parameters today. The impetus for this effort originates in the seemingly predictable improvement one obtains by training large models, the so-called scaling laws [KMH+20, HBM+22, MRB+23]. However these laws assume a \"fixed\" data source. This assumption is now significantly disrupted by the existence of frontier LLMs themselves, which allow us to interact with data in novel ways. In our previous works on the phi models [GZA+23, LBE+23, JBA+23] it was shown that a combination of LLM-based filtering of publicly available web data, and LLM-created synthetic data, enable performance in smaller language models that were typically seen only in much larger mod- els. For example our previous model trained on this data recipe, phi-2 (2.7B parameters), matched the performance of models 25 times larger trained on regular data. In this report we present a new model, phi-3-mini (3.8B parameters), trained for 3.3T tokens on larger and more advanced versions of the datasets used in phi-2. With its small size, phi-3-mini can easily be inferenced locally on a modern phone (see Figure 2), yet it achieves a quality that seems on-par with models such as Mixtral 8x7B [JSR+24] and GPT-3.5.\\n\\n<!-- PageNumber=\"1\" -->\\n\\n\\n## arXiv:2404.14219v3 [cs.CL] 23 May 2024\\n:selected: :unselected:\\nUser: Explain why it is surprising that one can build a language model small enough to fit on a phone, yet almost as powerful as ChatGPT. Just use one funny sentence.\\n\\nphi-3-mini: It\\'s like fitting a supercomputer in a flip phone, but instead of breaking the phone, it just breaks the internet with its tiny, yet mighty, linguistic prowess! User: Okay now more serious answer, and note that this was achieved solely by changing the training data.\\n\\nphi-3-mini: The development of a compact language model that rivals the capabilities of ChatGPT, while fitting on a phone, is a testament to the power of data-driven machine learning. By meticulously curating and optimizing the training dataset, researchers can significantly reduce the model\\'s size without compromising its performance. [ . . . ]\\n\\n\\n# 2 Technical Specifications\\n\\nThe phi-3-mini model is a transformer decoder architecture [VSP+17], with default context length 4K. We also introduce a long context version via LongRope [DZZ+24a] that extends the context length to 128K, called phi-3-mini-128K.\\n\\nTo best benefit the open source community, phi-3-mini is built upon a similar block structure as Llama-2 [TLI+23] and uses the same tokenizer with vocabulary size of 320641. This means that all packages developed for Llama-2 family of models can be directly adapted to phi-3-mini. The model uses 3072 hidden dimension, 32 heads and 32 layers. We trained using bfloat 16 for a total of 3.3T tokens. The model is already chat-finetuned, and the chat template is as follows:\\n\\n\\n# <|user | >\\\\\\\\n Question <|end|>\\\\\\\\n <|assistant | >\\n\\nThe phi-3-small model (7B parameters) leverages the tiktoken tokenizer (for better multilingual tokenization) with a vocabulary size of 1003522 and has default context length 8192. It follows the standard decoder architecture of a 7B model class, having 32 heads, 32 layers and a hidden size of 4096. We switched from GELU activation to GEGLU and used Maximal Update Parametrization (muP) [?] to tune hyperparameters on a small proxy model and transfer them to the target 7B model. Those helped ensure better performance and training stability. Also, the model leverages a grouped-query attention, with 4 queries sharing 1 key. To optimize the training and inference speed, we design a novel blocksparse attention module. For each attention head, the blocksparse attention enforces different sparsity patterns over KV cache. This ensures that all tokens are attended to on different heads for the given choice of sparsity. As illustrated in Figure 1, the context is then efficiently divided and conquered among attention heads, with significant KV cache reduction. To achieve actual deployment speed-up from the blocksparse design, we implemented highly efficient, yet flexible kernels for both training and inference. For training, we build a triton kernel based on Flash Attention [DFE+22]. For inference, we implemented a kernel for the prefilling phase and extended the paged attention kernel in vLLM for the decoding phase [KLZ+23]. Lastly, in phi-3-small architecture, we alternate dense attention layers and blocksparse attention layers to optimize KV cache savings while maintaining long context retrieval performance. An additional 10% multilingual data was also used for this model.\\n\\nHighly capable language model running locally on a cell-phone. Thanks to its small size, phi- 3-mini can be quantized to 4-bits so that it only occupies \\\\~ 1.8GB of memory. We tested the quantized model by deploying phi-3-mini on iPhone 14 with A16 Bionic chip running natively on-device and fully offline achieving more than 12 tokens per second.\\n\\n<!-- Footnote=\"1 We remove BoS tokens and add some additional tokens for chat template.\" -->\\n\\n<!-- Footnote=\"2We remove unused tokens from the vocabulary.\" -->\\n\\n<!-- PageNumber=\"2\" -->\\n\\n| | Block1 | Block2 | Block3 | Block4 | Block5 | Block6 | Block7 | Block8 |\\n| - | - | - | - | - | - | - | - | - |\\n| Head1 | | | | | | | | |\\n| Head2 | | | | | | | | |\\n| Head3 | | | | | | | | |\\n| Head4 | | | | | | | | |\\n\\nFigure 1: Toy illustration of the blocksparse attention in phi-3-small with 2 local blocks and vertical stride of 3. The table shows the Keys/values a query token in block 8 attended to. Blue=local blocks, orange=remote/vertical blocks, gray=blocks skipped.\\n\\nTraining Methodology. We follow the sequence of works initiated in \"Textbooks Are All You Need\" [GZA+23], which utilize high quality training data to improve the performance of small language models and deviate from the standard scaling-laws. In this work we show that such method allows to reach the level of highly capable models such as GPT-3.5 or Mixtral with only 3.8B total parameters (while Mixtral has 45B total parameters for example). Our training data of consists of heavily filtered publicly available web data (according to the \"educational level\") from various open internet sources, as well as synthetic LLM-generated data. Pre-training is performed in two disjoint and sequential phases; phase-1 comprises mostly of web sources aimed at teaching the model general knowledge and language understanding. Phase-2 merges even more heavily filtered webdata (a subset used in Phase-1) with some synthetic data that teach the model logical reasoning and various niche skills.\\n\\nData Optimal Regime. Unlike prior works that train language models in either \"compute optimal regime\" [HBM+22] or \"over-train regime\", we mainly focus on the quality of data for a given scale.3 We try to calibrate the training data to be closer to the \"data optimal\" regime for small models. In particular, we filter the publicly available web data to contain the correct level of \"knowledge\" and keep more web pages that could potentially improve the \"reasoning ability\" for the model. As an example, the result of a game in premier league in a particular day might be good training data for frontier models, but we need to remove such information to leave more model capacity for \"reasoning\" for the mini size models. We compare our approach with Llama-2 in Figure 3.\\n\\nTo test our data on larger size of models, we also trained phi-3-medium, a model with 14B pa- rameters using the same tokenizer and architecture of phi-3-mini, and trained on the same data for slightly more epochs (4.8T tokens total as for phi-3-small. The model has 40 heads and 40 layers, with embedding dimension 5120. We observe that some benchmarks improve much less from 7B to 14B than they do from 3.8B to 7B, perhaps indicating that our data mixture needs further work to be in the \"data optimal regime\" for 14B parameters model.\\n\\nPost-training. Post-training of phi-3-mini went through two stages, including supervised finetuning (SFT) and direct preference optimization (DPO). SFT leverages highly curated high-quality data across diverse domains, e.g., math, coding, reasoning, conversation, model identity, and safety. The SFT data mix starts with using English-only examples. DPO data covers chat format data, reasoning, and responsible AI (RAI) efforts. We use DPO to steer the model away from unwanted behavior, by using those outputs as \"rejected\" responses. Besides improvement in math, coding, reasoning, robustness, and safety, post-training transforms a language model to an AI assistant that users can efficiently and safely interact with.\\n\\n<!-- Footnote=\"3 Just like for \"compute optimal regime\", we use the term \"optimal\" in an aspirational sense for \"data optimal regime\". We are not implying that we actually found the provably \"optimal\" data mixture for a given scale.\" -->\\n\\n<!-- PageNumber=\"3\" -->\\n\\n<figure>\\n\\n![](figures/0)\\n\\n</figure>\\n\\n<figure>\\n\\n![](figures/1)\\n\\n<!-- FigureContent=\"Phi-3-mini-4k-instruct-q4 < < Phi-3-mini-4k-instruct-q4 < You You Phi-3-mini-4k-instruct-q4 Write a short poem that is hard to sing. What are some must do activities in Houston, TX area with 11 yr old kids? You LLM Write a title for the paper where each letter starts with P on very small language model that is more powerful than bigger ones. In whispers soft, a riddle\\'s dance, LLM Here are some must-do activities in the Houston, TX area that will be enjoyable for a 11-year old: Cryptic rhymes in convoluted stance. LLM A tongue twister\\'s treacherous lure, 1. Space Center Houston: This is an amazing space museum that offers numerous interactive exhibits and hands-on experiences related to astronautics, including moon rocks brought back from Apollo 17 mission, the International Space Station (ISS) simulator ride, and NASA\\'s shuttle program displays. Title: \"Pinnacle Potential: Powering Progressive Paradigms with Pristine Predictive Prowess\" Melodies tangled, hearts unsure. This title encapsulates the essence of a paper focused on an advanced, small-scale language model exhibiting extraordinary predictive capabilities. By using each letter to start successive words and highlighting its superior potential (\"Pinnacle Potential\"), it emphasizes the breakthrough nature of such technology compared to larger models (\"Powering Progressive Paradigms\"). The inclusion of \"with Pristine Predictive Prowess\" underscores the precise prediction ability that sets this language model apart. Through syllables that slyly tangle and knot, Echoing choruses of a labyrinthic thought, 2. Houston Zoo: The zoo has over 600 species of animals on display in a naturalistic environment. Take With pitches so high they baffle the ear, Type your message ... A sonnet\\'s challenge to vocal frontier. 1 The 10.66 ses, 12.47 t/s For in this poem where words entwine, Q WE RTYUIOP An enigmatic song that dares not align, A S D F G H J K L To sing may lead one down a perplexed trail, Z :selected: X In pursuit of rhythm, where melodies flail. CV B N M :selected: X 11.80 ses, 14.75 t/s 123 space return Type your message ... Type your message ... :)\" -->\\n\\n<figcaption>\\n\\nFigure 2: 4-bit quantized phi-3-mini running natively on an iPhone with A16 Bionic chip, generating over 12\\ntokens per second.\\n\\n</figcaption>\\n\\n</figure>\\n\\n\\n<figure>\\n\\n![](figures/2)\\n\\n<!-- FigureContent=\"Data Optimal Scaling Law versus Model families trained on the same data 66.69 Fitted line for Phi Fitted line for Llama2 54.60 MMLU error rate (percentage) 44.70 36.60 29.96 24.53 - 20.09 1.00 1.65 2.72 4.48 7.39 12.18 20.09 33.12 54.60 90.02 Model size (Billons)\" -->\\n\\n<figcaption>\\n\\nFigure 3: Scaling law close to the \"Data Optimal Regime\" (from left to right: phi-1.5, phi-2, phi-3-mini, phi-3-\\nsmall) versus Llama-2 family of models (7B, 13B, 34B, 70B) that were trained on the same fixed data. We plot\\nthe log of MMLU error versus the log of model size.\\n\\n</figcaption>\\n\\n</figure>\\n\\n\\n<!-- PageNumber=\"4\" -->\\n:unselected:\\nAs part of the post-training process, we developed a long context version of phi-3-mini with context length limit enlarged to 128K instead of 4K. Across the board, the 128K model quality is on par with the 4K length version, while being able to handle long context tasks. Long context extension has been done in two stages, including long context mid-training and long-short mixed post-training with both SFT and DPO.\\n\\n\\n# Academic benchmarks 3\\n\\nOn the next page we report the results for phi-3-mini on standard open-source benchmarks measuring the model\\'s reasoning ability (both common sense reasoning and logical reasoning). We compare to phi-2 [JBA+23], Mistral-7b-v0.1 [JSM+23], Mixtral-8x7b [JSR+24], Gemma 7B [TMH+24], Llama-3-instruct- 8b [AI], and GPT-3.5. All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable. These numbers might differ from other published numbers due to slightly different choices in the evaluation. As is now standard, we use few-shot prompts to evaluate the models, at temperature 0. The prompts and number of shots are part of a Microsoft internal tool to evaluate language models, and in particular we did no optimization to the pipeline for the phi-3 models.4 The number of k-shot examples is listed per-benchmark. An example of a 2-shot prompt is described in Appendix A.\\n\\n<!-- Footnote=\"4For example, we found that using ## before the Question can lead to a noticeable improvement to phi-3-mini\\'s results across many benchmarks, but we did not do such changes in the prompts.\" -->\\n\\n<!-- PageNumber=\"5\" -->\\n\\n| ||| Phi-3-mini Phi-3-small Phi-3-medium 3.8b  7b  14b || Phi-2 Mistral 2.7b  7b | Gemma 7b || Llama-3-In Mixtral 8b  8x7b | GPT-3.5 version 1106 |\\n| - | - | - | - | - | - | - | - | - | - |\\n| MMLU (5-Shot) [HBK+21] | 68.8 | 75.7 | 78.0 | 56.3 | 61.7 | 63.6 | 66.5 | 70.5 | 71.4 |\\n| HellaSwag (5-Shot) [ZHB+19] | 76.7 | 77.0 | 82.4 | 53.6 | 58.5 | 49.8 | 71.1 | 70.4 | 78.8 |\\n| ANLI (7-Shot) [NWD+20] | 52.8 | 58.1 | 55.8 | 42.5 | 47.1 | 48.7 | 57.3 | 55.2 | 58.1 |\\n| GSM-8K (8-Shot; CoT) [CKB+21] | 82.5 | 89.6 | 91.0 | 61.1 | 46.4 | 59.8 | 77.4 | 64.7 | 78.1 |\\n| MedQA (2-Shot) [JPO+20] AGIEval | 53.8 | 65.4 | 69.9 | 40.9 | 50.0 | 49.6 | 60.5 | 62.2 | 63.4 |\\n||||||\\n|| | | | | | 37.5 | 45.1 | 50.2 | 29.8 | 35.1 | 42.1 | 42.0 | 45.2 | 48.4 |\\n| (0-Shot) [ZCG+23] TriviaQA (5-Shot) [JCWZ17] |\\n|| | | | | | | | || 64.0 | 58.1 | 73.9 | 45.2 | 75.2 | 72.3 | 67.7 | 82.2 | 85.8 |\\n| Arc-C (10-Shot) [CCE+18] | 84.9 | 90.7 | 91.6 | 75.9 | 78.6 | 78.3 | 82.8 | 87.3 | 87.4 |\\n| Arc-E (10-Shot) [CCE+18] | 94.6 | 97.0 | 97.7 | 88.5 | 90.6 | 91.4 | 93.4 | 95.6 | 96.3 |\\n| PIQA | 84.2 | 86.9 | 87.9 | 60.2 | 77.7 | 78.1 | 75.7 | 86.0 | 86.6 |\\n| (5-Shot) [BZGC19] ||||||||||\\n| SociQA (5-Shot) [BZGC19] | 76.6 | 79.2 | 80.2 | 68.3 | 74.6 | 65.5 | 73.9 | 75.9 | 68.3 |\\n| BigBench-Hard (3-Shot; CoT) [SRR+22, SSS+22] | 71.7 | 79.1 | 81.4 | 59.4 | 57.3 | 59.6 | 51.5 | 69.7 | 68.32 |\\n| WinoGrande (5-Shot) [SLBBC19] | 70.8 | 81.5 | 81.5 | 54.7 | 54.2 | 55.6 | 65.0 | 62.0 | 68.8 |\\n| Open BookQA (10-Shot) [MCKS18] | 83.2 | 88.0 | 87.4 | 73.6 | 79.8 | 78.6 | 82.6 | 85.8 | 86.0 |\\n| BoolQ (2-Shot) [CLC+19] | 77.2 | 84.8 | 86.5 | – :unselected: | 72.2 | 66.0 | 80.9 | 77.6 | 79.1 |\\n| CommonSenseQA (10-Shot) [THLB19] TruthfulQA (10-Shot; MC2) [LHE22] | 80.2 | 80.0 | 82.8 | 69.3 | 72.6 | 76.2 | 79.0 | 78.1 | 79.6 |\\n|| 65.0 | 70.2 | 75.1 | – | 53.0 | 52.1 | 63.2 | 60.1 | 85.8 |\\n| HumanEval (0-Shot) [CTJ+21] | 58.5 | 61.0 | 62.2 | 59.0 | 28.0 | 34.1 | 60.4 | 37.8 | 62.2 |\\n| MBPP (3-Shot) [AON+21] | 70.0 | 71.7 | 75.2 | 60.6 | 50.8 | 51.5 | 67.7 | 60.2 | 77.8 |\\n| Average | 71.2 | 75.7 | 78.5 | – :unselected: | 61.2 | 61.7 | 69.4 | 69.8 | 74.3 |\\n| GPQA (2-Shot; CoT) [RHS+23] | 32.8 | 34.3 | – | – :unselected: | – :unselected: | – :unselected: :unselected: :unselected: | – :unselected: | – :unselected: :unselected: | 29.0 |\\n| MT Bench (2 round ave.) [ZCS+23] | 8.38 | 8.70 | 8.91 | – :unselected: :unselected: | – :unselected: :unselected: | – :unselected: | – :unselected: :unselected: | – :unselected: :unselected: | 8.35 |\\n\\n\\n# 4 Safety\\n\\nPhi-3-mini was developed in accordance with Microsoft\\'s responsible AI principles. The overall ap- proach consisted of safety alignment in post-training, red-teaming, automated testing and evaluations across dozens of RAI harm categories. Helpfulness and harmlessness preference datasets [BJN+22, JLD+23] with modifications inspired by [BSA+24] and multiple in-house generated datasets were lever- aged to address the RAI harm categories in safety post-training. An independent red team at Microsoft iteratively examined phi-3-mini to further identify areas of improvement during the post-training pro- cess. Based on their feedback, we curated additional datasets tailored to address their insights, thereby refining the post-training dataset. This process resulted in significant decrease of harmful response rates, as shown in Figure 4.\\n\\n<!-- PageNumber=\"6\" -->\\n<figure>\\n\\n![](figures/3)\\n\\n<!-- FigureContent=\"Comparison of Content Generated by Phi Text Models Before and After Safety Post-Training model Percentage (%) of Harmful Responses 70- :selected: Phi Text (without post-training) Phi Text (with post-training + red teaming) 60 - 50 - 40- 30 - 20 - 10 - 0 current events cyber fairness\\\\_bias hate\\\\_speech model\\\\_identity political\\\\_misinfo sexual violence Harm area\" -->\\n\\n<figcaption>\\n\\nFigure 4: Comparison of harmful response percentages by Microsoft AI Red Team between phi-3-mini before\\nand after the safety alignment. Note that the harmful response percentages in this chart are inflated numbers as\\nthe red team tried to induce phi-3-mini in an adversarial way to generate harmful responses through multi-turn\\nconversations.\\n\\n</figcaption>\\n\\n</figure>\\n\\n\\n| |||||| Phi-3-mini Phi-3-small Phi-3-medium Phi-2 Mistral Gemma 3.8b  7b  14b  2.7b  7b  7b | Llama-3-In 8b |\\n| - | - | - | - | - | - | - | - |\\n| Ungroundedness | 0.603 | 0.299 | 0.213 | 1.481 | 0.935 | 0.679 | 0.328 |\\n| Third Party Harm (DR-1) | 0.240 | 0.253 | 0.251 | 0.240 | 0.562 | 0.383 | 0.373 |\\n| Harmful Content Continuation (DR-3) | 0.007 | 0.003 | 0.010 | 0.029 | 0.026 | 0.013 | 0.013 |\\n| Harmful Content Summarization (DR-3) | 0.100 | 0.110 | 0.112 | 0.144 | 0.223 | 0.103 | 0.082 |\\n| Jailbreak (DR-1) | 0.123 | 0.107 | 0.111 | 0.150 | 0.156 | 0.114 | 0.130 |\\n\\nTable 1: Comparison of Microsoft internal multi-turn conversation RAI benchmark results of phi-3 models and other models. Note that a lower value indicates a better performance for all metrics in the table.\\n\\nThe safety alignment of phi-3-small and phi-3-medium was conducted by undergoing the same red-teaming process, utilizing identical datasets, and incorporating a slightly larger number of samples. Table 1 shows the results of in-house RAI benchmarks [MHJ+23] for phi-3 models compared to phi-2 [JBA+23], Mistral-7b-v0.1 [JSM+23], Gemma 7b [TMH+24], and Llama-3-instruct-8b [AI]. This bench- mark utilized GPT-4 to simulate multi-turn conversations in five different categories and to evaluate the model responses. Ungroundedness between 0 (fully grounded) and 4 (not grounded) measures if the information in a response is based on a given prompt. In other categories, responses were evaluated in terms of the severity of harmfulness from 0 (no harm) to 7 (extreme harm) and the defect rates (DR-x) were computed as the percentage of samples with the severity score being greater than or equal to x.\\n\\n\\n## 5\\n\\n\\n# Weakness\\n\\nIn terms of LLM capabilities, while phi-3-mini model achieves similar level of language understanding and reasoning ability as much larger models, it is still fundamentally limited by its size for certain tasks.\\n\\n<!-- PageNumber=\"7\" -->\\n\\n<!-- PageHeader=\"Suggest an itinerary for a 3-day trip during the 2026 Winter Olympic games\" -->\\n\\n<!-- PageHeader=\"Suggest an itinerary for a 3-day trip during the 2026 Winter Olympic games\" -->\\n :selected:\\n<figure>\\n\\n![](figures/4)\\n\\n<!-- FigureContent=\"Day 1: Arrival and Surrounding Neighborhoods Web Search Completed Morning: . Arrive in Pyeongchang, South Korea, and check-in to your accommodations. Day 1: Arrival and Exploring Milan · Enjoy a light breakfast at your hotel. . Arrive in Milan on February 6, 2026, the day of the Opening Ceremony. Mid-morning: . Spend the day exploring Milan\\'s city center, including Piazza del Duomo, Galleria Vittorio Emanuele II, and the Sforza Castle. . Take a leisurely walk around the town and explore the local shops and cafes. . Enjoy the vibrant Milanese nightlife, with options like the city\\'s many trendy cafes and restaurants. Lunch: . Visit the new Fiera Milano expo center for a unique and inspiring experience. · Try local specialties such as kimbap (Korean rice rolls) or mandu (Korean dumplings) at a nearby restaurant. Day 2: Cortina d\\'Ampezzo and Outdoor Activities Afternoon: . Take the train from Milan to Cortina d\\'Ampezzo, arriving in the afternoon. . Visit Pyeongchang Olympic Cauldron Park, the site of the 2018 Winter Olympics, to take photos and admire the beautiful scenery. . Check into your hotel and spend the evening relaxing and enjoying the local cuisine, such as risotto and polenta, at traditional pizzerias or trattorias. . Rent a bike and explore the surrounding neighborhoods, such as Pyeongchang Village, where you can find souvenir shops, outdoor activities, and various dining options. . On Day 2, take part in various outdoor activities, such as skiing, snowboarding, or tubing. Many of these activities are available at nearby resorts like Madonna di Campiglio, Kronplatz, or Passo San Pellegrino. . In the evening, explore Cortina d\\'Ampezzo\\'s charming streets, visit the historic St. Felix Castle, and enjoy the beautiful mountain scenery. Evening: . Dine at a local restaurant and enjoy traditional Korean cuisine such as samgyeopsal (grilled pork belly) or bulgogi (marinated beef). Day 3: Paralympic Games and Departure . On Day 3, attend the Paralympic Games, held in Turin on March 6, 2026. Day 2: Olympic Sports and Cultural Activities . Take the train from Cortina d\\'Ampezzo to Turin, arriving in time for the Paralympic events. Morning: . Explore Turin\\'s attractions, such as the Mole Antonelliana, the Egyptian Museum, and Piazza Vittorio Veneto. . Visit the Alpensia Sports Park, where the majority of the 2026 Winter Olympic events will be held. . See the ski slopes, ice rinks, and indoor arenas, and watch a glimpse of the opening and closing ceremonies. . Return to Milan in the evening for your departure, concluding your 3-day itinerary during the Milano Cortina 2026 Winter Olympic Games. Throughout your trip, make sure to book your accommodations, transportation, and tickets in advance, as these events are highly popular and may sell out quickly. Enjoy your time in Italy and the exciting Winter Olympic Games! Mid-morning: . Participate in a guided tour to learn about the history of the 2026 Winter Olympics and the unique features of the Pyeongchang venues. Sources: olympics.com olympics.com milanocortina2026.olympics.com :selected: milanwintertravel.com olympics.com :unselected: Search web @ Search web 1 Ask anything Ask anything\" -->\\n\\n<figcaption>\\n\\nFigure 5: Left: phi-3-mini\\'s completion without search. Right: phi-3-mini\\'s completion with search, using the\\ndefault HuggingFace Chat-UI search ability. For reference, the 2026 Winter Olympic Games are scheduled to be\\nheld in Milano and Cortina in Italy, while the 2022 and 2018 Winter Olympic Games were held in Beijing, China\\nand PyeongChang, Korea, respectively. Without the search results, the response is incorrect, while with the web\\nsearch, not only does the response become accurate, but also gets more specific with suggestions.\\n\\n</figcaption>\\n\\n</figure>\\n\\n\\nThe model simply does not have the capacity to store too much \"factual knowledge\", which can be seen for example with low performance on TriviaQA. However, we believe such weakness can be resolved by augmentation with a search engine. We show an example using the HuggingFace default Chat-UI with phi-3-mini in Figure 5. Another weakness related to model\\'s capacity is that we mostly restricted the language to English. Exploring multilingual capabilities for Small Language Models is an important next step, with some initial promising results on phi-3-small by including more multilingual data.\\n\\nDespite our diligent RAI efforts, as with most LLMs, there remains challenges around factual inaccu- racies (or hallucinations), reproduction or amplification of biases, inappropriate content generation, and safety issues. The use of carefully curated training data, and targeted post-training, and improvements from red-teaming insights significantly mitigates these issues across all dimensions. However, there is significant work ahead to fully address these challenges, and downstream use of the models should be evaluated for the specific use cases and safety considerations for that context.\\n\\n<!-- PageNumber=\"8\" -->\\n\\nextract all displayed texts and keep original displayed format. For the left section, please\\n<figure>\\n\\n![](figures/5)\\n\\n<!-- FigureContent=\"use markdown table with column \"coffee\", \"small\", \"median\" and \"large\". Instead, for the right section, please use JSON format. COFFEE & TEA O OSMMEDLG Teas CUP! POT Certainly! Below is the markdown table converted from the image: Esspresso 1.50 ,4.00 1 30 1 60 2.00 Americana 1.40 1.75 | 2.10 Cappucino 2.25 2 90 3.25 GREEN Latte 2.35 3 00 3.25 BLACK Mocha 12 35 3.00 3.50 WHITE Chai 2.25 2.40 3.25 OOLONG Chai Bomb 2.75 3.25 3.75 HERBAL House Cafe 1.50 1.75 2 00 \\\\* ADD .50 FOR HONEY Café au Lait 1.75 2 50 3.25 \\\\*ADD 1.00 FORICE And here is the JSON representation of the teas offered: \"Teas\": { What is the price of a Cappuccino with large size? \"CUP\": { \"GREEN\": \"$1.50\", \"BLACK\": \"$1.50\", :selected: The price for a Cappuccino in large size is $3.25. \"WHITE\": \"$1.50\", \"OOLONG\": \"$1.50\", \"HERBAL\": \"$1.50\" How much does it cost to add ice to a tea? \"POT\": { \"GREEN\": \"$4.00\", \"BLACK\": \"$4.00\", :selected: . It costs an additional $1.00 to add ice to any tea. \"WHITE\": \"$4.00\", \"OOLONG\": \"$4.00\", \"HERBAL\": \"$4.00\" If someone wants to buy a pot of tea, how much would it cost? \"\\\\*ADD .50 FOR HONEY\\\\*\", \"\\\\*ADD $1.00 FORICE\\\\*\" . A pot of tea would cost $4.00. 7\" -->\\n\\n<figcaption>\\n\\nFigure 6: The demo case shows Phi-3-Vision\\'s capability in natural image understanding and reasoning.\\n\\n</figcaption>\\n\\n</figure>\\n\\n\\n\\n# 6 Phi-3-Vision\\n\\n\\n## 6.1 Technical Specifications\\n\\nArchitecture The Phi-3-Vision (4.2B parameters) is a multimodal model designed to process an image and a textual prompt as inputs, and subsequently generate textual outputs. This model is com- posed of two primary components: an image encoder, i.e., CLIP ViT-L/14 [RKH+21] and a transformer decoder, i.e., phi-3-mini-128K-instruct. The visual tokens, once extracted by the image encoder, are then combined with text tokens in an interleaved way (no particular order for image and text tokens). To ac- commodate high-resolution images and various aspect ratios, a dynamic cropping strategy [DZZ+24b] is utilized to split the input image into a 2d array of blocks, where the tokens of the blocks are concatenated to represent the whole image.\\n\\nPre-training The Phi-3-Vision model undergoes a pre-training phase using a diverse dataset, which consists of a combination of interleaved image-text documents (e.g., [LST+24]), image-text pairs from FLD-5B [XWX+24], synthetic data derived from Optical Character Recognition (OCR) of PDF files, datasets for chart/table comprehension, and text-only data. The objective of predicting the next token is employed specifically on text tokens, while any loss associated with image tokens is disregarded during this phase. The pre-training process involves a total of 0.5T tokens that encompass both visual and text elements. During the pre-training phase, the maximum image resolution is capped at 1344 x 1344 as the majority of the training images are smaller than this resolution.\\n\\n<!-- PageNumber=\"9\" -->\\n\\n| \\\\| Coffee 1 - ||| \\\\| Small \\\\| Medium \\\\| Large   |\\n| - | - | - | - |\\n| \\\\| Espresso | \\\\| 1.30| \\\\| 1.60  | 2.00 |\\n| \\\\| Americana | \\\\| 1.40 | \\\\| 1.75| \\\\| 2.10  |\\n| \\\\| Cappuccino | \\\\| 2.25 | \\\\| 2.90| \\\\| 3.25  |\\n| \\\\| Latte | \\\\| 2.35 | \\\\| 3.00| \\\\| 3.25  |\\n| \\\\| Mocha | \\\\| 2.35 | \\\\| 3.00| \\\\| 3.50  |\\n| \\\\| Chai | \\\\| 2.25 | \\\\| 2.40 | \\\\| 3.25 |\\n| \\\\| Chai Bomb | \\\\| 2.75 | \\\\| 3.25| \\\\| 3.75  |\\n| \\\\| House Cafe | \\\\| 1.50 | \\\\| 1.75| \\\\| 2.00  |\\n| \\\\| Cafe au Lait | \\\\| 1.75 | \\\\| 2.50| \\\\| 3.25  |\\n:selected:\\nPost-training. The Phi-3-Vision model contains two post-training stages: supervised finetuning (SFT) and direct preference optimization (DPO). For SFT, we leveraged text SFT dataset, public mul- timodal instruct tuning datasets along with large-scale multimodal instruct tuning datasets that we built ourselves, covering diverse domains and tasks such as general natural image understanding, chart/table/- diagram understanding/reasoning, PowerPoint understanding, and model safety. The multimodal SFT data has about a total of 15B tokens. For DPO we mainly use a text DPO dataset and a relatively smaller-scale multimodal DPO dataset. For these two stages, we jointly train multimodal tasks and text- only tasks so that the model can achieve multi-modal reasoning while maintaining language capabilities as much as possible.\\n\\n\\n## 6.2 Academic benchmarks\\n\\nWe report in Table 2 the evaluation results of Phi-3-Vision on nine open-source academic benchmarks. These benchmarks evaluate reasoning and perceptual capabilities on visual and text inputs and can be grouped in three categories: Science, Charts, and Generic knowledge. We compare Phi-3-Vision with the following baselines: MM1-3B-Chat [MGF+24], MM1-7B-Chat [MGF+24], Llava-1.6 Vicuna 7B [LLLL23], Llava-1.6 Llama3-8B [LLL+24], Qwen-VL-Chat [BBY+23], Claude 3 Haiku [Ant24], Gemini 1.0 Pro V [TAB+23], and GPT-4V-Turbo. Our performance quality assessment setup used the same evaluation pipeline for all the baselines to ensure a fair comparison, with the exception of MM1-3B-Chat. We just copied and pasted their published numbers since the model is not publicly available.\\n\\nOur evaluation setup aimed to mimic scenarios where regular users interact with a multi-modal model, i.e., users who are not experts in prompt engineering or know special techniques that can improve performance. For this reason, we adopted the evaluation setting used in Llava-1.5 [LLLL23]. In this setup, the prompts include instructions to select a single letter corresponding to an answer from a list of given options, or answer with a single word or phrase. In our prompts, we did not use specific tokens for multiple-choice questions. Moreover, we did not scale or pre-process any image in our benchmarking system. We placed the images as the first item in the prompts, except on the MMMU dataset where the prompts interleave the images anywhere in the question or the answers. Lastly, our evaluation setup only considered a 0-shot format. Because of these evaluation parameters, our reported numbers can differ from the published numbers of the considered baselines.\\n\\n\\n## 6.3 Safety\\n\\nTo ensure the integration of Phi-3-Vision aligns with Microsoft\\'s Responsible AI (RAI) principles, we involved safety post-training in both Supervised Fine-Tuning (SFT) stage and Direct Preference Optimization (DPO) stage. In creating the safety training datasets, we utilized not only the text- only RAI datasets, but also a variety of in-house Multi-Modal (MM) RAI datasets that cover various harm categories identified in both public and internal MM RAI benchmarks. For the purpose of RAI evaluation, we performed a rigorous quantitative assessment on both public and internal benchmarks, this was done in conjunction with a human evaluation conducted by Microsoft\\'s internal red team.\\n\\nIn Table 3, we present the evaluation outcomes of Phi-3-Vision on three MM RAI benchmarks: one internal and two public benchmarks (specifically, RTVLM [LLY+24] and VLGuard [ZBY+24]). We juxtapose these results with those of other open-source models such as Llava-1.5 [LLLL23], Llava-1.6 [LLL+24], Qwen-VL-Chat [BBY+23], and GPT4-V[Ope23]. The results clearly indicate that safety post- training notably enhances the RAI performance of Phi-3-Vision across all RAI benchmarks. In Figure 7, we further breakdown the performance across different RAI categories of the VLGuard and Internal benchmarks, demonstrating that safety post-training can aid Phi-3-Vision in improving RAI performance in nearly all categories.\\n\\n<!-- PageNumber=\"10\" -->\\n\\n| | 4.2b Phi-3-Vision | MM1-3B-Chat 3.6b [MGF+24] | 7.6b [MGF+24] MM1-7B-Chat | LLaVA-1.6 Vicuna-7b [LLLL23] | LLaVA-Next LLama3-8b [LLL+24] | 9.6b [BBY+23] Qwen-VL-Chat | [Ant24] Claude 3 haiku | [TAB+23] Gemini 1.0 Pro V | GPT-4V-Turbo turbo-2024-04-09 |\\n| - | - | - | - | - | - | - | - | - | - |\\n| MMMU (val) [YNZ+23] | 40.4 | 33.9 | 37.0 | 34.2 | 36.4 | 39.0 | 40.7 | 42.0 | 55.5 |\\n| ScienceQA (test) [LMX+22] | 90.8 | 69.4 | 72.6 | 70.6 | 73.7 | 67.2 | 72.0 | 79.7 | 75.7 |\\n| Math Vista (testmini) [LBX+24] Inter-GPS (test) [LGJ+21] | 44.5 | 32.0 | 35.9 | 31.5 | 34.8 | 29.4 | 33.2 | 35.0 | 47.5 |\\n|| 38.1 | - | - | 20.5 | 24.6 | 22.3 | 32.1 | 28.6 | 41.0 |\\n| MMBench (dev-en) [LDZ+24] POPE (test) [LDZ+23] | 80.5 | 75.9 | 79.0 | 76.3 | 79.4 | 75.8 | 62.4 | 80.0 | 86.1 |\\n|| 85.8 | 87.4 | 86.6 | 87.2 | 87.0 | 82.6 | 74.4 | 84.2 | 83.7 |\\n| AI2D (test) [KSK+16] ChartQA (test) [MLT+22] | 76.7 | - | - | 63.1 | 66.9 | 59.8 | 60.3 | 62.8 | 74.7 |\\n|| 81.4 | - | - | 55.0 | 65.8 | 50.9 | 59.3 | 58.0 | 62.3 |\\n| Text VQA (test) [SNS+19] | 70.9 | 71.9 | 72.8 | 64.6 | 55.7 | 59.4 | 62.7 | 64.7 | 68.1 |\\n\\nTable 2: Comparison results on public MLLM benchmarks. All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat [MGF+24] and MM1-7B- Chat [MGF+24], which are not publicly available. We adopted the evaluation setting used in Llava-1.5 [LLLL23], without any specific prompt or pre-processing image for all results. These numbers might differ from other published numbers due to slightly different prompts.\\n\\n\\n## 6.4 Weakness\\n\\nRegarding the multi-modal LLM capabilities of our Phi-3-Vision, it performs admirably across various fields. However, we have identified certain limitations, particularly with questions necessitating high- level reasoning abilities. Additionally, the model has been observed to occasionally generate ungrounded outputs, making it potentially unreliable in sensitive areas, such as finance. To mitigate these issues, we will incorporate more reasoning-focused and hallucination-related DPO data into post-training in the future.\\n\\nFrom a responsible AI standpoint, whilst safety post-training has made significant strides, our Phi- 3-Vision occasionally fails to refrain from answering harmful or sensitive inquiries. Examples of such occasions include deciphering particular types of captcha and describing scam images containing disin- formation or hallucination. We find that this issue partly arises from the capabilities, such as OCR, acquired during the training process with normal instruct tuning datasets, which can be regarded as the trade-off between helpfulness and harmlessness. Moving forward, we need to further explore this area to achieve a better balance.\\n\\n<!-- PageNumber=\"11\" -->\\n\\n| |||| Phi-3-Vision Phi-3-Vision w/o safety Llava-1.6 Vicuna Qwen-VL-Chat 3.8b+0.3b  3.8b+0.3b  7b+0.3b  7.7b+1.9b | GPT4-V N/A |\\n| - | - | - | - | - | - |\\n| Internal (private) | 8.30 | 7.06 | 5.44 | 7.27 | 8.55 |\\n| RTVLM (public) | 4.64 | 3.56 | 3.86 | 4.78 | 6.81 |\\n| VLGuard (public) | 9.12 | 4.66 | 5.62 | 8.33 | 8.90 |\\n\\nTable 3: Comparison results on public and private multi-modal RAI benchmarks. Note that all metrics in the table are [0,10] and a higher value indicates a better performance.\\n\\n<figure>\\n\\n![](figures/6)\\n\\n<!-- FigureContent=\"Phi-3-Vision w/o safety Phi-3-Vision Phi-3-Vision w/o safety Phi-3-Vision 10 10 8 8 6 6 4 4 2 2 0 0 safeimg-safetxt deception discrimination privacy risk-behaviour safeimg-unsafetxt Total harmful jailbreak privacy grounding Total\" -->\\n\\n<figcaption>\\n\\nFigure 7: Comparison of categorized RAI performance of Phi-3-Vision with and without the safety post-training\\non the VLGuard (left) and Internal (right) benchmark, respectively. It clearly indicates that safety post-training\\ncan enhance the RAI performance across nearly all the RAI categories.\\n\\n</figcaption>\\n\\n</figure>\\n\\n\\n\\n# References\\n\\n[AI] Meta AI. Introducing meta llama 3: The most capable openly available llm to date.\\n\\n[Ant24] AI Anthropic. The claude 3 model family: Opus, sonnet, haiku. Claude-3 Model Card, 2024.\\n\\n[AON+21] Jacob Austin, Augustus Odena, Maxwell Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie Cai, Michael Terry, Quoc Le, and Charles Sutton. Program synthesis with large language models. arXiv preprint ar Xiv:2108.07732, 2021.\\n\\n[BBY+23] Jinze Bai, Shuai Bai, Shusheng Yang, Shijie Wang, Sinan Tan, Peng Wang, Junyang Lin, Chang Zhou, and Jingren Zhou. Qwen-vl: A versatile vision-language model for understand- ing, localization, text reading, and beyond, 2023.\\n\\n[BJN+22] Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, Nicholas Joseph, Saurav Kada- vath, Jackson Kernion, Tom Conerly, Sheer El-Showk, Nelson Elhage, Zac Hatfield-Dodds, Danny Hernandez, Tristan Hume, Scott Johnston, Shauna Kravec, Liane Lovitt, Neel Nanda, Catherine Olsson, Dario Amodei, Tom Brown, Jack Clark, Sam McCandlish, Chris Olah, Ben Mann, and Jared Kaplan. Training a helpful and harmless assistant with reinforcement learning from human feedback, 2022.\\n\\n<!-- PageNumber=\"12\" -->\\n\\n[BSA+24] Federico Bianchi, Mirac Suzgun, Giuseppe Attanasio, Paul Röttger, Dan Jurafsky, Tatsunori Hashimoto, and James Zou. Safety-tuned llamas: Lessons from improving the safety of large language models that follow instructions, 2024.\\n\\n[BZGC19] Yonatan Bisk, Rowan Zellers, Jianfeng Gao, and Yejin Choi. Piqa: Reasoning about physical commonsense in natural language. arXiv preprint arXiv:1911.11641, 2019.\\n\\n[CCE+18] Peter Clark, Isaac Cowhey, Oren Etzioni, Tushar Khot, Ashish Sabharwal, Carissa Schoenick, and Oyvind Tafjord. Think you have solved question answering? try arc, the ai2 reasoning challenge, 2018.\\n\\n[CKB+21] Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, Christopher Hesse, and John Schulman. Training verifiers to solve math word problems. ar Xiv preprint ar Xiv:2110.14168, 2021.\\n\\n[CLC+19] Christopher Clark, Kenton Lee, Ming-Wei Chang, Tom Kwiatkowski, Michael Collins, and Kristina Toutanova. Boolq: Exploring the surprising difficulty of natural yes/no questions. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pages 2924-2936, 2019.\\n\\n[CTJ+21] Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mo- hammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, and Wojciech Zaremba. Evaluating large language models trained on code, 2021.\\n\\n[DFE+22] Tri Dao, Dan Fu, Stefano Ermon, Atri Rudra, and Christopher Re. Flashattention: Fast and memory-efficient exact attention with io-awareness. Advances in Neural Information Processing Systems, 35:16344-16359, 2022.\\n\\n[DZZ+24a] Yiran Ding, Li Lyna Zhang, Chengruidong Zhang, Yuanyuan Xu, Ning Shang, Jiahang Xu, Fan Yang, and Mao Yang. Longrope: Extending llm context window beyond 2 million tokens, 2024.\\n\\n[DZZ+24b] Xiaoyi Dong, Pan Zhang, Yuhang Zang, Yuhang Cao, Bin Wang, Linke Ouyang, Songyang Zhang, Haodong Duan, Wenwei Zhang, Yining Li, et al. Internlm-xcomposer2-4khd: A pioneering large vision-language model handling resolutions from 336 pixels to 4k hd. arXiv preprint ar Xiv:2404.06512, 2024.\\n\\n[GZA+23] Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Gustavo de Rosa Piero Kauffmann, Olli Saarikivia,\\n\\n<!-- PageNumber=\"13\" -->\\n\\nAdil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen El- dan, Adam Tauman Kalai, Yin Tat Lee, and Yuanzhi Li. Textbooks are all you need. arXiv preprint ar Xiv:2306.11644, 2023.\\n\\n[HBK+21] Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, and Jacob Steinhardt. Measuring mathematical problem solving with the MATH dataset, 2021.\\n\\n[HBM+22] Jordan Hoffmann, Sebastian Borgeaud, Arthur Mensch, Elena Buchatskaya, Eliza Ruther- ford Trevor Cai, Diego de Las Casas, Lisa Anne Hendricks, Johannes Welbl, Aidan Clark, Tom Hennigan, Eric Noland, Katie Millican, George van den Driessche, Bogdan Damoc, Au- relia Guy, Simon Osindero, Karen Simonyan, Erich Elsen, Jack W. Rae, Oriol Vinyals, and Laurent Sifre. Training compute-optimal large language models. arXiv preprint ar Xiv:2203.15556, 2022.\\n\\n[JBA+23] Mojan Javaheripi, Sebastien Bubeck, Marah Abdin, Jyoti Aneja, Caio Cesar Teodoro Mendes, Weizhu Chen, Allie Del Giorno, Ronen Eldan, Sivakanth Gopi, Suriya Gunasekar, Piero Kauffmann, Yin Tat Lee, Yuanzhi Li, Anh Nguyen, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Michael Santacroce, Harkirat Singh Behl, Adam Tau- mann Kalai, Xin Wang, Rachel Ward, Philipp Witte, Cyril Zhang, and Yi Zhang. Phi-2: The surprising power of small language models. Microsoft Research Blog, 2023.\\n\\n[JCWZ17] Mandar Joshi, Eunsol Choi, Daniel S. Weld, and Luke Zettlemoyer. Triviaqa: A large scale distantly supervised challenge dataset for reading comprehension, 2017.\\n\\n[JLD+23] Jiaming Ji, Mickel Liu, Juntao Dai, Xuehai Pan, Chi Zhang, Ce Bian, Chi Zhang, Ruiyang Sun, Yizhou Wang, and Yaodong Yang. Beavertails: Towards improved safety alignment of llm via a human-preference dataset, 2023.\\n\\n[JPO+20] Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits. What disease does this patient have? a large-scale open domain question answering dataset from medical exams, 2020.\\n\\n[JSM+23] Albert Q. Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, Lélio Renard Lavaud, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William El Sayed. Mistral 7b, 2023.\\n\\n[JSR+24] Albert Q. Jiang, Alexandre Sablayrolles, Antoine Roux, Arthur Mensch, Blanche Savary, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Emma Bou Hanna, Florian Bressand, Gianna Lengyel, Guillaume Bour, Guillaume Lample, Lélio Renard Lavaud, Lu- cile Saulnier, Marie-Anne Lachaux, Pierre Stock, Sandeep Subramanian, Sophia Yang, Szy- mon Antoniak, Teven Le Scao, Théophile Gervet, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William El Sayed. Mixtral of experts, 2024.\\n\\n[KLZ+23] Woosuk Kwon, Zhuohan Li, Siyuan Zhuang, Ying Sheng, Lianmin Zheng, Cody Hao Yu, Joseph E. Gonzalez, Hao Zhang, and Ion Stoica. Efficient memory management for large language model serving with pagedattention. In Proceedings of the ACM SIGOPS 29th Symposium on Operating Systems Principles, 2023.\\n\\n<!-- PageNumber=\"14\" -->\\n\\n[KMH+20] Jared Kaplan, Sam McCandlish, Tom Henighan, Tom B Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. Scaling laws for neural language models. arXiv preprint ar Xiv:2001.08361, 2020.\\n\\n[KSK+16] Aniruddha Kembhavi, Mike Salvato, Eric Kolve, Minjoon Seo, Hannaneh Hajishirzi, and Ali Farhadi. A diagram is worth a dozen images, 2016.\\n\\n[LBE+23] Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar, and Yin Tat Lee. Textbooks are all you need ii: phi-1.5 technical report. arXiv preprint ar Xiv:2309.05463, 2023.\\n\\n[LBX+24] Pan Lu, Hritik Bansal, Tony Xia, Jiacheng Liu, Chunyuan Li, Hannaneh Hajishirzi, Hao Cheng, Kai-Wei Chang, Michel Galley, and Jianfeng Gao. Mathvista: Evaluating mathe- matical reasoning of foundation models in visual contexts, 2024.\\n\\n[LDZ+23] Yifan Li, Yifan Du, Kun Zhou, Jinpeng Wang, Wayne Xin Zhao, and Ji-Rong Wen. Evalu- ating object hallucination in large vision-language models, 2023.\\n\\n[LDZ+24] Yuan Liu, Haodong Duan, Yuanhan Zhang, Bo Li, Songyang Zhang, Wangbo Zhao, Yike Yuan, Jiaqi Wang, Conghui He, Ziwei Liu, Kai Chen, and Dahua Lin. Mmbench: Is your multi-modal model an all-around player?, 2024.\\n\\n[LGJ+21] Pan Lu, Ran Gong, Shibiao Jiang, Liang Qiu, Siyuan Huang, Xiaodan Liang, and Song-Chun Zhu. Inter-gps: Interpretable geometry problem solving with formal language and symbolic reasoning, 2021.\\n\\n[LHE22] Stephanie Lin, Jacob Hilton, and Owain Evans. Truthfulqa: Measuring how models mimic human falsehoods, 2022.\\n\\n[LLL+24] Haotian Liu, Chunyuan Li, Yuheng Li, Bo Li, Yuanhan Zhang, Sheng Shen, and Yong Jae Lee. Llava-next: Improved reasoning, ocr, and world knowledge, January 2024.\\n\\n[LLLL23] Haotian Liu, Chunyuan Li, Yuheng Li, and Yong Jae Lee. Improved baselines with visual instruction tuning. arXiv preprint ar Xiv:2310.03744, 2023.\\n\\n[LLY+24] Mukai Li, Lei Li, Yuwei Yin, Masood Ahmed, Zhenguang Liu, and Qi Liu. Red teaming visual language models. ar Xiv preprint ar Xiv:2401.12915, 2024.\\n\\n[LMX+22] Pan Lu, Swaroop Mishra, Tony Xia, Liang Qiu, Kai-Wei Chang, Song-Chun Zhu, Oyvind Tafjord, Peter Clark, and Ashwin Kalyan. Learn to explain: Multimodal reasoning via thought chains for science question answering. In The 36th Conference on Neural Information Processing Systems (NeurIPS), 2022.\\n\\n[LST+24] Hugo Laurencon, Lucile Saulnier, Leo Tronchon, Stas Bekman, Amanpreet Singh, An- ton Lozhkov, Thomas Wang, Siddharth Karamcheti, Alexander Rush, Douwe Kiela, et al. Obelics: An open web-scale filtered dataset of interleaved image-text documents. Advances in Neural Information Processing Systems, 36, 2024.\\n\\n[MCKS18] Todor Mihaylov, Peter Clark, Tushar Khot, and Ashish Sabharwal. Can a suit of armor conduct electricity? a new dataset for open book question answering, 2018.\\n\\n<!-- PageNumber=\"15\" -->\\n\\n[MGF+24] Brandon McKinzie, Zhe Gan, Jean-Philippe Fauconnier, Sam Dodge, Bowen Zhang, Philipp Dufter, Dhruti Shah, Xianzhi Du, Futang Peng, Floris Weers, Anton Belyi, Haotian Zhang, Karanjeet Singh, Doug Kang, Ankur Jain, Hongyu Hè, Max Schwarzer, Tom Gunter, Xiang Kong, Aonan Zhang, Jianyu Wang, Chong Wang, Nan Du, Tao Lei, Sam Wiseman, Guoli Yin, Mark Lee, Zirui Wang, Ruoming Pang, Peter Grasch, Alexander Toshev, and Yinfei Yang. Mm1: Methods, analysis & insights from multimodal llm pre-training. ar Xiv preprint ar Xiv:2403.09611, 2024.\\n\\n[MHJ+23] Ahmed Magooda, Alec Helyar, Kyle Jackson, David Sullivan, Chad Atalla, Emily Sheng, Dan Vann, Richard Edgar, Hamid Palangi, Roman Lutz, Hongliang Kong, Vincent Yun, Eslam Kamal, Federico Zarfati, Hanna Wallach, Sarah Bird, and Mei Chen. A framework for automated measurement of responsible ai harms in generative ai applications, 2023.\\n\\n[MLT+22] Ahmed Masry, Do Long, Jia Qing Tan, Shafiq Joty, and Enamul Hoque. ChartQA: A benchmark for question answering about charts with visual and logical reasoning. In Findings of the Association for Computational Linguistics: ACL 2022, pages 2263-2279, Dublin, Ireland, May 2022. Association for Computational Linguistics.\\n\\n[MRB+23] Niklas Muennighoff, Alexander M Rush, Boaz Barak, Teven Le Scao, Aleksandra Piktus, Nouamane Tazi, Sampo Pyysalo, Thomas Wolf, and Colin Raffel. Scaling data-constrained language models. arXiv preprint arXiv:2305.16264, 2023.\\n\\n[NWD+20] Yixin Nie, Adina Williams, Emily Dinan, Mohit Bansal, Jason Weston, and Douwe Kiela. Adversarial nli: A new benchmark for natural language understanding, 2020.\\n\\n[Ope23] OpenAI. Gpt-4v(ision) system card, 2023. https://cdn.openai.com/papers/GPTV\\\\_ System\\\\_Card. pdf.\\n\\n[RHS+23] David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, and Samuel R. Bowman. Gpqa: A graduate-level google-proof q&a benchmark, 2023.\\n\\n[RKH+21] Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agar- wal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et al. Learning transferable visual models from natural language supervision. In International conference on machine learning, pages 8748-8763. PMLR, 2021.\\n\\n[RWC+19] Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, and Ilya Sutskever. Language models are unsupervised multitask learners. OpenAI blog, 1(8):9, 2019.\\n\\n[SLBBC19] Keisuke Sakaguchi, Ronan Le Bras, Chandra Bhagavatula, and Yejin Choi. Winogrande: An adversarial winograd schema challenge at scale. arXiv preprint arXiv:1907.10641, 2019.\\n\\n[SNS+19] Amanpreet Singh, Vivek Natarajan, Meet Shah, Yu Jiang, Xinlei Chen, Dhruv Batra, Devi Parikh, and Marcus Rohrbach. Towards vqa models that can read, 2019.\\n\\n[SRR+22] Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adrià Garriga-Alonso, et al. Beyond the imitation game: Quantifying and extrapolating the capabilities of language models. arXiv preprint ar Xiv:2206.04615, 2022.\\n\\n<!-- PageNumber=\"16\" -->\\n\\n[SSS+22] Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, and Jason Wei. Chal- lenging big-bench tasks and whether chain-of-thought can solve them, 2022.\\n\\n[TAB+23] Gemini Team, Rohan Anil, Sebastian Borgeaud, Yonghui Wu, Jean-Baptiste Alayrac, Jiahui Yu, Radu Soricut, Johan Schalkwyk, Andrew M Dai, Anja Hauth, et al. Gemini: a family of highly capable multimodal models. arXiv preprint ar Xiv:2312.11805, 2023.\\n\\n[THLB19] Alon Talmor, Jonathan Herzig, Nicholas Lourie, and Jonathan Berant. Commonsenseqa: A question answering challenge targeting commonsense knowledge, 2019.\\n\\n[TLI+23] Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, and Guillaume Lample. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971, 2023.\\n\\n[TMH+24] Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivière, Mihir Sanjay Kale, Juliette Love, et al. Gemma: Open models based on gemini research and technology, 2024.\\n\\n[VSP+17] Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, L ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In Advances in Neural Information Processing Systems, volume 30, 2017.\\n\\n[XWX+24] Bin Xiao, Haiping Wu, Weijian Xu, Xiyang Dai, Houdong Hu, Yumao Lu, Michael Zeng, Ce Liu, and Lu Yuan. Florence-2: Advancing a unified representation for a variety of vision tasks. 2024.\\n\\n[YNZ+23] Xiang Yue, Yuansheng Ni, Kai Zhang, Tianyu Zheng, Ruoqi Liu, Ge Zhang, Samuel Stevens, Dongfu Jiang, Weiming Ren, Yuxuan Sun, Cong Wei, Botao Yu, Ruibin Yuan, Renliang Sun, Ming Yin, Boyuan Zheng, Zhenzhu Yang, Yibo Liu, Wenhao Huang, Huan Sun, Yu Su, and Wenhu Chen. Mmmu: A massive multi-discipline multimodal understanding and reasoning benchmark for expert agi, 2023.\\n\\n[ZBY+24] Yongshuo Zong, Ondrej Bohdal, Tingyang Yu, Yongxin Yang, and Timothy Hospedales. Safety fine-tuning at (almost) no cost: A baseline for vision large language models. arXiv preprint ar Xiv:2402.02207, 2024.\\n\\n[ZCG+23] Wanjun Zhong, Ruixiang Cui, Yiduo Guo, Yaobo Liang, Shuai Lu, Yanlin Wang, Amin Saied, Weizhu Chen, and Nan Duan. Agieval: A human-centric benchmark for evaluating foundation models, 2023.\\n\\n[ZCS+23] Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi Lin, Zhuohan Li, Dacheng Li, Eric Xing, et al. Judging llm-as-a-judge with mt-bench and chatbot arena. arXiv preprint ar Xiv:2306.05685, 2023.\\n\\n[ZHB+19] Rowan Zellers, Ari Holtzman, Yonatan Bisk, Ali Farhadi, and Yejin Choi. Hellaswag: Can a machine really finish your sentence? In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pages 4791-4800, 2019.\\n\\n<!-- PageNumber=\"17\" -->\\n\\n# A Example prompt for benchmarks\\n\\nQuestion:\\n\\nSolve for x: (-3)(-4-3x)=2\\n\\nOptions :\\n\\nA. - 5 6\\n\\nB. 7\\n\\n6\\n\\nC. 5\\n\\n3\\n\\nD.\\n\\n1\\n\\nAnswer: A\\n\\nQuestion: Which of the following is the body cavity that contains the pituitary gland? Options :\\n\\nA. Abdominal\\n\\nB. Cranial\\n\\nC. Pleural\\n\\nD. Spinal\\n\\nAnswer: B\\n\\nQuestion: Where was the most famous site of the mystery cults in Greece? Options :\\n\\nA. Ephesus\\n\\nB. Corinth\\n\\nC. Athens\\n\\nD. Eleusis\\n\\nAnswer:\\n\\n<!-- PageNumber=\"18\" -->\\n\\n# B Authors (alphabetical)\\n\\n||||\\n| - | - | - |\\n| Marah Abdin | Jamie Huynh | Olli Saarikivi |\\n| Sam Ade Jacobs | Dan Iter | Amin Saied |\\n| Ammar Ahmad Awan | Russell J. Hewett | Adil Salim |\\n| Jyoti Aneja | Mojan Javaheripi | Michael Santacroce |\\n| Hany Awadalla | Xin Jin | Shital Shah |\\n| Ahmed Awadallah | Nikos Karampatziakis | Ning Shang |\\n| Nguyen Bach | Piero Kauffmann | Hiteshi Sharma |\\n| Amit Bahree | Mahoud Khademi | Swadheen Shukla |\\n| Arash Bakhtiari | Dongwoo Kim | Xia Song |\\n| Jianmin Bao | Lev Kurilenko | Yin Tat Lee |\\n| Harkirat Behl | Yuanzhi Li | Andrea Tupini |\\n| Alon Benhaim | Yunsheng Li | Xin Wang |\\n| Misha Bilenko | Chen Liang | Guanhua Wang |\\n| Johan Bjorck | Lars Liden | Lijuan Wang |\\n| Sébastien Bubeck | Eric Lin | Chunyu Wang :unselected: |\\n| Caio César Teodoro Mendes | Zeqi Lin | Yu Wang |\\n| Martin Cai | Weishung Liu | Rachel Ward |\\n| Qin Cai | Mengchen Liu | Wen Wen |\\n| Vishrav Chaudhary | Ce Liu | Philipp Witte |\\n| Weizhu Chen | Chong Luo :unselected: | Haiping Wu |\\n| Yi-Ling Chen | Li Lyna Zhang | Michael Wyatt |\\n| Yen-Chun Chen | Piyush Madan | Bin Xiao |\\n| Dongdong Chen | David Majercak | Jiahang Xu |\\n| Dong Chen | Matt Mazzola | Can Xu |\\n| Parul Chopra | Arindam Mitra | Weijian Xu |\\n| Xiyang Dai | Hardik Modi | Sonali Yadav |\\n| Gustavo de Rosa | Anh Nguyen | Fan Yang |\\n| Allie Del Giorno | Brandon Norick | Ziyi Yang |\\n| Matthew Dixon | Barun Patra | Jianwei Yang |\\n| Ronen Eldan | Daniel Perez-Becker | Yifan Yang |\\n| Victor Fragoso | Thomas Portet | Donghan Yu :unselected: |\\n| Mei Gao | Reid Pryzant | Lu Yuan |\\n| Min Gao | Heyang Qin | Chengruidong Zhang :unselected: |\\n| Jianfeng Gao | James R. Lee | Cyril Zhang :unselected: |\\n| Abhishek Goswami | Marko Radmilac | Jianwen Zhang |\\n| Suriya Gunasekar | Corby Rosset | Yi Zhang |\\n| Emman Haider | Sambudha Roy | Yunan Zhang |\\n| Junheng Hao | Olatunji Ruwase | Xiren Zhou |\\n\\n\\n# C Acknowledgements\\n\\nWe would like to thank Zhuohan Li, Simon Mo, and Kaichao You from UC Berkeley for sharing their insights on the vLLM kernel.\\n\\n<!-- PageNumber=\"19\" -->\\n:unselected:')]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e2e2397",
   "metadata": {},
   "source": [
    "### Semantic chunking\n",
    "\n",
    "**Markdown** is a structured and formatted markup language and a popular input for enabling semantic chunking in RAG (Retrieval-Augmented Generation). You can use the Markdown content from the Layout model to split documents based on paragraph boundaries, create specific chunks for tables, and fine-tune your chunking strategy to improve the quality of the generated responses.\n",
    "\n",
    "This method divides the text into chunks based on **semantic understanding**. Division boundaries are focused on sentence subject and use significant computational algorithmically complex resources. However, it has the distinct advantage of maintaining semantic consistency within each chunk. It's useful for text summarization, sentiment analysis, and document classification tasks.\n",
    "\n",
    "https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept-retrieval-augmented-generation?view=doc-intel-4.0.0#semantic-chunking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d6a1e99e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of chunks = 17\n"
     ]
    }
   ],
   "source": [
    "headers_to_split_on = [\n",
    "    (\"#\", \"Header 1\"),\n",
    "    (\"##\", \"Header 2\"),\n",
    "    (\"###\", \"Header 3\"),\n",
    "    (\"####\", \"Header 4\"),\n",
    "]\n",
    "\n",
    "text_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)\n",
    "docs_string = documents[0].page_content\n",
    "chunks = text_splitter.split_text(docs_string)\n",
    "\n",
    "print(f\"Length of chunks = {len(chunks)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3dde97d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Chunk 1 *****\n",
      "Length of chunk = 93\n",
      "\n",
      "\u001b[1;34mPhi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone\n",
      "===  \n",
      "Microsoft\u001b[0m\n",
      "\n",
      "***** Chunk 2 *****\n",
      "Length of chunk = 1028\n",
      "\n",
      "\u001b[1;34mWe introduce phi-3-mini, a 3.8 billion parameter language model trained on 3.3 trillion tokens, whose overall performance, as measured by both academic benchmarks and internal testing, rivals that of models such as Mixtral 8x7B and GPT-3.5 (e.g., phi-3-mini achieves 69% on MMLU and 8.38 on MT-bench), despite being small enough to be deployed on a phone. The innovation lies entirely in our dataset for training, a scaled-up version of the one used for phi-2, composed of heavily filtered publicly available web data and synthetic data. The model is also further aligned for robustness, safety, and chat format. We also provide some initial parameter-scaling results with a 7B and 14B models trained for 4.8T tokens, called phi-3-small and phi-3-medium, both significantly more capable than phi-3-mini (e.g., respectively 75% and 78% on MMLU, and 8.7 and 8.9 on MT-bench). Moreover, we also introduce phi-3-vision, a 4.2 billion parameter model based on phi-3-mini with strong reasoning capabilities for image and text prompts.\u001b[0m\n",
      "\n",
      "***** Chunk 3 *****\n",
      "Length of chunk = 1553\n",
      "\n",
      "\u001b[1;34mThe striking progress of AI in the last few years can be largely attributed to major efforts through- out the world towards scaling-up to ever-larger models and datasets. Large Language Models (LLMs) have steadily increased in size from a mere billion parameters just five years ago (GPT-2 had 1.5 bil- lion parameters [RWC+19]) to trillion parameters today. The impetus for this effort originates in the seemingly predictable improvement one obtains by training large models, the so-called scaling laws [KMH+20, HBM+22, MRB+23]. However these laws assume a \"fixed\" data source. This assumption is now significantly disrupted by the existence of frontier LLMs themselves, which allow us to interact with data in novel ways. In our previous works on the phi models [GZA+23, LBE+23, JBA+23] it was shown that a combination of LLM-based filtering of publicly available web data, and LLM-created synthetic data, enable performance in smaller language models that were typically seen only in much larger mod- els. For example our previous model trained on this data recipe, phi-2 (2.7B parameters), matched the performance of models 25 times larger trained on regular data. In this report we present a new model, phi-3-mini (3.8B parameters), trained for 3.3T tokens on larger and more advanced versions of the datasets used in phi-2. With its small size, phi-3-mini can easily be inferenced locally on a modern phone (see Figure 2), yet it achieves a quality that seems on-par with models such as Mixtral 8x7B [JSR+24] and GPT-3.5.  \n",
      "<!-- PageNumber=\"1\" -->\u001b[0m\n",
      "\n",
      "***** Chunk 4 *****\n",
      "Length of chunk = 821\n",
      "\n",
      "\u001b[1;34m:selected: :unselected:\n",
      "User: Explain why it is surprising that one can build a language model small enough to fit on a phone, yet almost as powerful as ChatGPT. Just use one funny sentence.  \n",
      "phi-3-mini: It's like fitting a supercomputer in a flip phone, but instead of breaking the phone, it just breaks the internet with its tiny, yet mighty, linguistic prowess! User: Okay now more serious answer, and note that this was achieved solely by changing the training data.  \n",
      "phi-3-mini: The development of a compact language model that rivals the capabilities of ChatGPT, while fitting on a phone, is a testament to the power of data-driven machine learning. By meticulously curating and optimizing the training dataset, researchers can significantly reduce the model's size without compromising its performance. [ . . . ]\u001b[0m\n",
      "\n",
      "***** Chunk 5 *****\n",
      "Length of chunk = 703\n",
      "\n",
      "\u001b[1;34mThe phi-3-mini model is a transformer decoder architecture [VSP+17], with default context length 4K. We also introduce a long context version via LongRope [DZZ+24a] that extends the context length to 128K, called phi-3-mini-128K.  \n",
      "To best benefit the open source community, phi-3-mini is built upon a similar block structure as Llama-2 [TLI+23] and uses the same tokenizer with vocabulary size of 320641. This means that all packages developed for Llama-2 family of models can be directly adapted to phi-3-mini. The model uses 3072 hidden dimension, 32 heads and 32 layers. We trained using bfloat 16 for a total of 3.3T tokens. The model is already chat-finetuned, and the chat template is as follows:\u001b[0m\n",
      "\n",
      "***** Chunk 6 *****\n",
      "Length of chunk = 9500\n",
      "\n",
      "\u001b[1;34mThe phi-3-small model (7B parameters) leverages the tiktoken tokenizer (for better multilingual tokenization) with a vocabulary size of 1003522 and has default context length 8192. It follows the standard decoder architecture of a 7B model class, having 32 heads, 32 layers and a hidden size of 4096. We switched from GELU activation to GEGLU and used Maximal Update Parametrization (muP) [?] to tune hyperparameters on a small proxy model and transfer them to the target 7B model. Those helped ensure better performance and training stability. Also, the model leverages a grouped-query attention, with 4 queries sharing 1 key. To optimize the training and inference speed, we design a novel blocksparse attention module. For each attention head, the blocksparse attention enforces different sparsity patterns over KV cache. This ensures that all tokens are attended to on different heads for the given choice of sparsity. As illustrated in Figure 1, the context is then efficiently divided and conquered among attention heads, with significant KV cache reduction. To achieve actual deployment speed-up from the blocksparse design, we implemented highly efficient, yet flexible kernels for both training and inference. For training, we build a triton kernel based on Flash Attention [DFE+22]. For inference, we implemented a kernel for the prefilling phase and extended the paged attention kernel in vLLM for the decoding phase [KLZ+23]. Lastly, in phi-3-small architecture, we alternate dense attention layers and blocksparse attention layers to optimize KV cache savings while maintaining long context retrieval performance. An additional 10% multilingual data was also used for this model.  \n",
      "Highly capable language model running locally on a cell-phone. Thanks to its small size, phi- 3-mini can be quantized to 4-bits so that it only occupies \\~ 1.8GB of memory. We tested the quantized model by deploying phi-3-mini on iPhone 14 with A16 Bionic chip running natively on-device and fully offline achieving more than 12 tokens per second.  \n",
      "<!-- Footnote=\"1 We remove BoS tokens and add some additional tokens for chat template.\" -->  \n",
      "<!-- Footnote=\"2We remove unused tokens from the vocabulary.\" -->  \n",
      "<!-- PageNumber=\"2\" -->  \n",
      "| | Block1 | Block2 | Block3 | Block4 | Block5 | Block6 | Block7 | Block8 |\n",
      "| - | - | - | - | - | - | - | - | - |\n",
      "| Head1 | | | | | | | | |\n",
      "| Head2 | | | | | | | | |\n",
      "| Head3 | | | | | | | | |\n",
      "| Head4 | | | | | | | | |  \n",
      "Figure 1: Toy illustration of the blocksparse attention in phi-3-small with 2 local blocks and vertical stride of 3. The table shows the Keys/values a query token in block 8 attended to. Blue=local blocks, orange=remote/vertical blocks, gray=blocks skipped.  \n",
      "Training Methodology. We follow the sequence of works initiated in \"Textbooks Are All You Need\" [GZA+23], which utilize high quality training data to improve the performance of small language models and deviate from the standard scaling-laws. In this work we show that such method allows to reach the level of highly capable models such as GPT-3.5 or Mixtral with only 3.8B total parameters (while Mixtral has 45B total parameters for example). Our training data of consists of heavily filtered publicly available web data (according to the \"educational level\") from various open internet sources, as well as synthetic LLM-generated data. Pre-training is performed in two disjoint and sequential phases; phase-1 comprises mostly of web sources aimed at teaching the model general knowledge and language understanding. Phase-2 merges even more heavily filtered webdata (a subset used in Phase-1) with some synthetic data that teach the model logical reasoning and various niche skills.  \n",
      "Data Optimal Regime. Unlike prior works that train language models in either \"compute optimal regime\" [HBM+22] or \"over-train regime\", we mainly focus on the quality of data for a given scale.3 We try to calibrate the training data to be closer to the \"data optimal\" regime for small models. In particular, we filter the publicly available web data to contain the correct level of \"knowledge\" and keep more web pages that could potentially improve the \"reasoning ability\" for the model. As an example, the result of a game in premier league in a particular day might be good training data for frontier models, but we need to remove such information to leave more model capacity for \"reasoning\" for the mini size models. We compare our approach with Llama-2 in Figure 3.  \n",
      "To test our data on larger size of models, we also trained phi-3-medium, a model with 14B pa- rameters using the same tokenizer and architecture of phi-3-mini, and trained on the same data for slightly more epochs (4.8T tokens total as for phi-3-small. The model has 40 heads and 40 layers, with embedding dimension 5120. We observe that some benchmarks improve much less from 7B to 14B than they do from 3.8B to 7B, perhaps indicating that our data mixture needs further work to be in the \"data optimal regime\" for 14B parameters model.  \n",
      "Post-training. Post-training of phi-3-mini went through two stages, including supervised finetuning (SFT) and direct preference optimization (DPO). SFT leverages highly curated high-quality data across diverse domains, e.g., math, coding, reasoning, conversation, model identity, and safety. The SFT data mix starts with using English-only examples. DPO data covers chat format data, reasoning, and responsible AI (RAI) efforts. We use DPO to steer the model away from unwanted behavior, by using those outputs as \"rejected\" responses. Besides improvement in math, coding, reasoning, robustness, and safety, post-training transforms a language model to an AI assistant that users can efficiently and safely interact with.  \n",
      "<!-- Footnote=\"3 Just like for \"compute optimal regime\", we use the term \"optimal\" in an aspirational sense for \"data optimal regime\". We are not implying that we actually found the provably \"optimal\" data mixture for a given scale.\" -->  \n",
      "<!-- PageNumber=\"3\" -->  \n",
      "<figure>  \n",
      "![](figures/0)  \n",
      "</figure>  \n",
      "<figure>  \n",
      "![](figures/1)  \n",
      "<!-- FigureContent=\"Phi-3-mini-4k-instruct-q4 < < Phi-3-mini-4k-instruct-q4 < You You Phi-3-mini-4k-instruct-q4 Write a short poem that is hard to sing. What are some must do activities in Houston, TX area with 11 yr old kids? You LLM Write a title for the paper where each letter starts with P on very small language model that is more powerful than bigger ones. In whispers soft, a riddle's dance, LLM Here are some must-do activities in the Houston, TX area that will be enjoyable for a 11-year old: Cryptic rhymes in convoluted stance. LLM A tongue twister's treacherous lure, 1. Space Center Houston: This is an amazing space museum that offers numerous interactive exhibits and hands-on experiences related to astronautics, including moon rocks brought back from Apollo 17 mission, the International Space Station (ISS) simulator ride, and NASA's shuttle program displays. Title: \"Pinnacle Potential: Powering Progressive Paradigms with Pristine Predictive Prowess\" Melodies tangled, hearts unsure. This title encapsulates the essence of a paper focused on an advanced, small-scale language model exhibiting extraordinary predictive capabilities. By using each letter to start successive words and highlighting its superior potential (\"Pinnacle Potential\"), it emphasizes the breakthrough nature of such technology compared to larger models (\"Powering Progressive Paradigms\"). The inclusion of \"with Pristine Predictive Prowess\" underscores the precise prediction ability that sets this language model apart. Through syllables that slyly tangle and knot, Echoing choruses of a labyrinthic thought, 2. Houston Zoo: The zoo has over 600 species of animals on display in a naturalistic environment. Take With pitches so high they baffle the ear, Type your message ... A sonnet's challenge to vocal frontier. 1 The 10.66 ses, 12.47 t/s For in this poem where words entwine, Q WE RTYUIOP An enigmatic song that dares not align, A S D F G H J K L To sing may lead one down a perplexed trail, Z :selected: X In pursuit of rhythm, where melodies flail. CV B N M :selected: X 11.80 ses, 14.75 t/s 123 space return Type your message ... Type your message ... :)\" -->  \n",
      "<figcaption>  \n",
      "Figure 2: 4-bit quantized phi-3-mini running natively on an iPhone with A16 Bionic chip, generating over 12\n",
      "tokens per second.  \n",
      "</figcaption>  \n",
      "</figure>  \n",
      "<figure>  \n",
      "![](figures/2)  \n",
      "<!-- FigureContent=\"Data Optimal Scaling Law versus Model families trained on the same data 66.69 Fitted line for Phi Fitted line for Llama2 54.60 MMLU error rate (percentage) 44.70 36.60 29.96 24.53 - 20.09 1.00 1.65 2.72 4.48 7.39 12.18 20.09 33.12 54.60 90.02 Model size (Billons)\" -->  \n",
      "<figcaption>  \n",
      "Figure 3: Scaling law close to the \"Data Optimal Regime\" (from left to right: phi-1.5, phi-2, phi-3-mini, phi-3-\n",
      "small) versus Llama-2 family of models (7B, 13B, 34B, 70B) that were trained on the same fixed data. We plot\n",
      "the log of MMLU error versus the log of model size.  \n",
      "</figcaption>  \n",
      "</figure>  \n",
      "<!-- PageNumber=\"4\" -->\n",
      ":unselected:\n",
      "As part of the post-training process, we developed a long context version of phi-3-mini with context length limit enlarged to 128K instead of 4K. Across the board, the 128K model quality is on par with the 4K length version, while being able to handle long context tasks. Long context extension has been done in two stages, including long context mid-training and long-short mixed post-training with both SFT and DPO.\u001b[0m\n",
      "\n",
      "***** Chunk 7 *****\n",
      "Length of chunk = 3670\n",
      "\n",
      "\u001b[1;34mOn the next page we report the results for phi-3-mini on standard open-source benchmarks measuring the model's reasoning ability (both common sense reasoning and logical reasoning). We compare to phi-2 [JBA+23], Mistral-7b-v0.1 [JSM+23], Mixtral-8x7b [JSR+24], Gemma 7B [TMH+24], Llama-3-instruct- 8b [AI], and GPT-3.5. All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable. These numbers might differ from other published numbers due to slightly different choices in the evaluation. As is now standard, we use few-shot prompts to evaluate the models, at temperature 0. The prompts and number of shots are part of a Microsoft internal tool to evaluate language models, and in particular we did no optimization to the pipeline for the phi-3 models.4 The number of k-shot examples is listed per-benchmark. An example of a 2-shot prompt is described in Appendix A.  \n",
      "<!-- Footnote=\"4For example, we found that using ## before the Question can lead to a noticeable improvement to phi-3-mini's results across many benchmarks, but we did not do such changes in the prompts.\" -->  \n",
      "<!-- PageNumber=\"5\" -->  \n",
      "| ||| Phi-3-mini Phi-3-small Phi-3-medium 3.8b  7b  14b || Phi-2 Mistral 2.7b  7b | Gemma 7b || Llama-3-In Mixtral 8b  8x7b | GPT-3.5 version 1106 |\n",
      "| - | - | - | - | - | - | - | - | - | - |\n",
      "| MMLU (5-Shot) [HBK+21] | 68.8 | 75.7 | 78.0 | 56.3 | 61.7 | 63.6 | 66.5 | 70.5 | 71.4 |\n",
      "| HellaSwag (5-Shot) [ZHB+19] | 76.7 | 77.0 | 82.4 | 53.6 | 58.5 | 49.8 | 71.1 | 70.4 | 78.8 |\n",
      "| ANLI (7-Shot) [NWD+20] | 52.8 | 58.1 | 55.8 | 42.5 | 47.1 | 48.7 | 57.3 | 55.2 | 58.1 |\n",
      "| GSM-8K (8-Shot; CoT) [CKB+21] | 82.5 | 89.6 | 91.0 | 61.1 | 46.4 | 59.8 | 77.4 | 64.7 | 78.1 |\n",
      "| MedQA (2-Shot) [JPO+20] AGIEval | 53.8 | 65.4 | 69.9 | 40.9 | 50.0 | 49.6 | 60.5 | 62.2 | 63.4 |\n",
      "||||||\n",
      "|| | | | | | 37.5 | 45.1 | 50.2 | 29.8 | 35.1 | 42.1 | 42.0 | 45.2 | 48.4 |\n",
      "| (0-Shot) [ZCG+23] TriviaQA (5-Shot) [JCWZ17] |\n",
      "|| | | | | | | | || 64.0 | 58.1 | 73.9 | 45.2 | 75.2 | 72.3 | 67.7 | 82.2 | 85.8 |\n",
      "| Arc-C (10-Shot) [CCE+18] | 84.9 | 90.7 | 91.6 | 75.9 | 78.6 | 78.3 | 82.8 | 87.3 | 87.4 |\n",
      "| Arc-E (10-Shot) [CCE+18] | 94.6 | 97.0 | 97.7 | 88.5 | 90.6 | 91.4 | 93.4 | 95.6 | 96.3 |\n",
      "| PIQA | 84.2 | 86.9 | 87.9 | 60.2 | 77.7 | 78.1 | 75.7 | 86.0 | 86.6 |\n",
      "| (5-Shot) [BZGC19] ||||||||||\n",
      "| SociQA (5-Shot) [BZGC19] | 76.6 | 79.2 | 80.2 | 68.3 | 74.6 | 65.5 | 73.9 | 75.9 | 68.3 |\n",
      "| BigBench-Hard (3-Shot; CoT) [SRR+22, SSS+22] | 71.7 | 79.1 | 81.4 | 59.4 | 57.3 | 59.6 | 51.5 | 69.7 | 68.32 |\n",
      "| WinoGrande (5-Shot) [SLBBC19] | 70.8 | 81.5 | 81.5 | 54.7 | 54.2 | 55.6 | 65.0 | 62.0 | 68.8 |\n",
      "| Open BookQA (10-Shot) [MCKS18] | 83.2 | 88.0 | 87.4 | 73.6 | 79.8 | 78.6 | 82.6 | 85.8 | 86.0 |\n",
      "| BoolQ (2-Shot) [CLC+19] | 77.2 | 84.8 | 86.5 | – :unselected: | 72.2 | 66.0 | 80.9 | 77.6 | 79.1 |\n",
      "| CommonSenseQA (10-Shot) [THLB19] TruthfulQA (10-Shot; MC2) [LHE22] | 80.2 | 80.0 | 82.8 | 69.3 | 72.6 | 76.2 | 79.0 | 78.1 | 79.6 |\n",
      "|| 65.0 | 70.2 | 75.1 | – | 53.0 | 52.1 | 63.2 | 60.1 | 85.8 |\n",
      "| HumanEval (0-Shot) [CTJ+21] | 58.5 | 61.0 | 62.2 | 59.0 | 28.0 | 34.1 | 60.4 | 37.8 | 62.2 |\n",
      "| MBPP (3-Shot) [AON+21] | 70.0 | 71.7 | 75.2 | 60.6 | 50.8 | 51.5 | 67.7 | 60.2 | 77.8 |\n",
      "| Average | 71.2 | 75.7 | 78.5 | – :unselected: | 61.2 | 61.7 | 69.4 | 69.8 | 74.3 |\n",
      "| GPQA (2-Shot; CoT) [RHS+23] | 32.8 | 34.3 | – | – :unselected: | – :unselected: | – :unselected: :unselected: :unselected: | – :unselected: | – :unselected: :unselected: | 29.0 |\n",
      "| MT Bench (2 round ave.) [ZCS+23] | 8.38 | 8.70 | 8.91 | – :unselected: :unselected: | – :unselected: :unselected: | – :unselected: | – :unselected: :unselected: | – :unselected: :unselected: | 8.35 |\u001b[0m\n",
      "\n",
      "***** Chunk 8 *****\n",
      "Length of chunk = 3355\n",
      "\n",
      "\u001b[1;34mPhi-3-mini was developed in accordance with Microsoft's responsible AI principles. The overall ap- proach consisted of safety alignment in post-training, red-teaming, automated testing and evaluations across dozens of RAI harm categories. Helpfulness and harmlessness preference datasets [BJN+22, JLD+23] with modifications inspired by [BSA+24] and multiple in-house generated datasets were lever- aged to address the RAI harm categories in safety post-training. An independent red team at Microsoft iteratively examined phi-3-mini to further identify areas of improvement during the post-training pro- cess. Based on their feedback, we curated additional datasets tailored to address their insights, thereby refining the post-training dataset. This process resulted in significant decrease of harmful response rates, as shown in Figure 4.  \n",
      "<!-- PageNumber=\"6\" -->\n",
      "<figure>  \n",
      "![](figures/3)  \n",
      "<!-- FigureContent=\"Comparison of Content Generated by Phi Text Models Before and After Safety Post-Training model Percentage (%) of Harmful Responses 70- :selected: Phi Text (without post-training) Phi Text (with post-training + red teaming) 60 - 50 - 40- 30 - 20 - 10 - 0 current events cyber fairness\\_bias hate\\_speech model\\_identity political\\_misinfo sexual violence Harm area\" -->  \n",
      "<figcaption>  \n",
      "Figure 4: Comparison of harmful response percentages by Microsoft AI Red Team between phi-3-mini before\n",
      "and after the safety alignment. Note that the harmful response percentages in this chart are inflated numbers as\n",
      "the red team tried to induce phi-3-mini in an adversarial way to generate harmful responses through multi-turn\n",
      "conversations.  \n",
      "</figcaption>  \n",
      "</figure>  \n",
      "| |||||| Phi-3-mini Phi-3-small Phi-3-medium Phi-2 Mistral Gemma 3.8b  7b  14b  2.7b  7b  7b | Llama-3-In 8b |\n",
      "| - | - | - | - | - | - | - | - |\n",
      "| Ungroundedness | 0.603 | 0.299 | 0.213 | 1.481 | 0.935 | 0.679 | 0.328 |\n",
      "| Third Party Harm (DR-1) | 0.240 | 0.253 | 0.251 | 0.240 | 0.562 | 0.383 | 0.373 |\n",
      "| Harmful Content Continuation (DR-3) | 0.007 | 0.003 | 0.010 | 0.029 | 0.026 | 0.013 | 0.013 |\n",
      "| Harmful Content Summarization (DR-3) | 0.100 | 0.110 | 0.112 | 0.144 | 0.223 | 0.103 | 0.082 |\n",
      "| Jailbreak (DR-1) | 0.123 | 0.107 | 0.111 | 0.150 | 0.156 | 0.114 | 0.130 |  \n",
      "Table 1: Comparison of Microsoft internal multi-turn conversation RAI benchmark results of phi-3 models and other models. Note that a lower value indicates a better performance for all metrics in the table.  \n",
      "The safety alignment of phi-3-small and phi-3-medium was conducted by undergoing the same red-teaming process, utilizing identical datasets, and incorporating a slightly larger number of samples. Table 1 shows the results of in-house RAI benchmarks [MHJ+23] for phi-3 models compared to phi-2 [JBA+23], Mistral-7b-v0.1 [JSM+23], Gemma 7b [TMH+24], and Llama-3-instruct-8b [AI]. This bench- mark utilized GPT-4 to simulate multi-turn conversations in five different categories and to evaluate the model responses. Ungroundedness between 0 (fully grounded) and 4 (not grounded) measures if the information in a response is based on a given prompt. In other categories, responses were evaluated in terms of the severity of harmfulness from 0 (no harm) to 7 (extreme harm) and the defect rates (DR-x) were computed as the percentage of samples with the severity score being greater than or equal to x.\u001b[0m\n",
      "\n",
      "***** Chunk 9 *****\n",
      "Length of chunk = 6956\n",
      "\n",
      "\u001b[1;34mIn terms of LLM capabilities, while phi-3-mini model achieves similar level of language understanding and reasoning ability as much larger models, it is still fundamentally limited by its size for certain tasks.  \n",
      "<!-- PageNumber=\"7\" -->  \n",
      "<!-- PageHeader=\"Suggest an itinerary for a 3-day trip during the 2026 Winter Olympic games\" -->  \n",
      "<!-- PageHeader=\"Suggest an itinerary for a 3-day trip during the 2026 Winter Olympic games\" -->\n",
      ":selected:\n",
      "<figure>  \n",
      "![](figures/4)  \n",
      "<!-- FigureContent=\"Day 1: Arrival and Surrounding Neighborhoods Web Search Completed Morning: . Arrive in Pyeongchang, South Korea, and check-in to your accommodations. Day 1: Arrival and Exploring Milan · Enjoy a light breakfast at your hotel. . Arrive in Milan on February 6, 2026, the day of the Opening Ceremony. Mid-morning: . Spend the day exploring Milan's city center, including Piazza del Duomo, Galleria Vittorio Emanuele II, and the Sforza Castle. . Take a leisurely walk around the town and explore the local shops and cafes. . Enjoy the vibrant Milanese nightlife, with options like the city's many trendy cafes and restaurants. Lunch: . Visit the new Fiera Milano expo center for a unique and inspiring experience. · Try local specialties such as kimbap (Korean rice rolls) or mandu (Korean dumplings) at a nearby restaurant. Day 2: Cortina d'Ampezzo and Outdoor Activities Afternoon: . Take the train from Milan to Cortina d'Ampezzo, arriving in the afternoon. . Visit Pyeongchang Olympic Cauldron Park, the site of the 2018 Winter Olympics, to take photos and admire the beautiful scenery. . Check into your hotel and spend the evening relaxing and enjoying the local cuisine, such as risotto and polenta, at traditional pizzerias or trattorias. . Rent a bike and explore the surrounding neighborhoods, such as Pyeongchang Village, where you can find souvenir shops, outdoor activities, and various dining options. . On Day 2, take part in various outdoor activities, such as skiing, snowboarding, or tubing. Many of these activities are available at nearby resorts like Madonna di Campiglio, Kronplatz, or Passo San Pellegrino. . In the evening, explore Cortina d'Ampezzo's charming streets, visit the historic St. Felix Castle, and enjoy the beautiful mountain scenery. Evening: . Dine at a local restaurant and enjoy traditional Korean cuisine such as samgyeopsal (grilled pork belly) or bulgogi (marinated beef). Day 3: Paralympic Games and Departure . On Day 3, attend the Paralympic Games, held in Turin on March 6, 2026. Day 2: Olympic Sports and Cultural Activities . Take the train from Cortina d'Ampezzo to Turin, arriving in time for the Paralympic events. Morning: . Explore Turin's attractions, such as the Mole Antonelliana, the Egyptian Museum, and Piazza Vittorio Veneto. . Visit the Alpensia Sports Park, where the majority of the 2026 Winter Olympic events will be held. . See the ski slopes, ice rinks, and indoor arenas, and watch a glimpse of the opening and closing ceremonies. . Return to Milan in the evening for your departure, concluding your 3-day itinerary during the Milano Cortina 2026 Winter Olympic Games. Throughout your trip, make sure to book your accommodations, transportation, and tickets in advance, as these events are highly popular and may sell out quickly. Enjoy your time in Italy and the exciting Winter Olympic Games! Mid-morning: . Participate in a guided tour to learn about the history of the 2026 Winter Olympics and the unique features of the Pyeongchang venues. Sources: olympics.com olympics.com milanocortina2026.olympics.com :selected: milanwintertravel.com olympics.com :unselected: Search web @ Search web 1 Ask anything Ask anything\" -->  \n",
      "<figcaption>  \n",
      "Figure 5: Left: phi-3-mini's completion without search. Right: phi-3-mini's completion with search, using the\n",
      "default HuggingFace Chat-UI search ability. For reference, the 2026 Winter Olympic Games are scheduled to be\n",
      "held in Milano and Cortina in Italy, while the 2022 and 2018 Winter Olympic Games were held in Beijing, China\n",
      "and PyeongChang, Korea, respectively. Without the search results, the response is incorrect, while with the web\n",
      "search, not only does the response become accurate, but also gets more specific with suggestions.  \n",
      "</figcaption>  \n",
      "</figure>  \n",
      "The model simply does not have the capacity to store too much \"factual knowledge\", which can be seen for example with low performance on TriviaQA. However, we believe such weakness can be resolved by augmentation with a search engine. We show an example using the HuggingFace default Chat-UI with phi-3-mini in Figure 5. Another weakness related to model's capacity is that we mostly restricted the language to English. Exploring multilingual capabilities for Small Language Models is an important next step, with some initial promising results on phi-3-small by including more multilingual data.  \n",
      "Despite our diligent RAI efforts, as with most LLMs, there remains challenges around factual inaccu- racies (or hallucinations), reproduction or amplification of biases, inappropriate content generation, and safety issues. The use of carefully curated training data, and targeted post-training, and improvements from red-teaming insights significantly mitigates these issues across all dimensions. However, there is significant work ahead to fully address these challenges, and downstream use of the models should be evaluated for the specific use cases and safety considerations for that context.  \n",
      "<!-- PageNumber=\"8\" -->  \n",
      "extract all displayed texts and keep original displayed format. For the left section, please\n",
      "<figure>  \n",
      "![](figures/5)  \n",
      "<!-- FigureContent=\"use markdown table with column \"coffee\", \"small\", \"median\" and \"large\". Instead, for the right section, please use JSON format. COFFEE & TEA O OSMMEDLG Teas CUP! POT Certainly! Below is the markdown table converted from the image: Esspresso 1.50 ,4.00 1 30 1 60 2.00 Americana 1.40 1.75 | 2.10 Cappucino 2.25 2 90 3.25 GREEN Latte 2.35 3 00 3.25 BLACK Mocha 12 35 3.00 3.50 WHITE Chai 2.25 2.40 3.25 OOLONG Chai Bomb 2.75 3.25 3.75 HERBAL House Cafe 1.50 1.75 2 00 \\* ADD .50 FOR HONEY Café au Lait 1.75 2 50 3.25 \\*ADD 1.00 FORICE And here is the JSON representation of the teas offered: \"Teas\": { What is the price of a Cappuccino with large size? \"CUP\": { \"GREEN\": \"$1.50\", \"BLACK\": \"$1.50\", :selected: The price for a Cappuccino in large size is $3.25. \"WHITE\": \"$1.50\", \"OOLONG\": \"$1.50\", \"HERBAL\": \"$1.50\" How much does it cost to add ice to a tea? \"POT\": { \"GREEN\": \"$4.00\", \"BLACK\": \"$4.00\", :selected: . It costs an additional $1.00 to add ice to any tea. \"WHITE\": \"$4.00\", \"OOLONG\": \"$4.00\", \"HERBAL\": \"$4.00\" If someone wants to buy a pot of tea, how much would it cost? \"\\*ADD .50 FOR HONEY\\*\", \"\\*ADD $1.00 FORICE\\*\" . A pot of tea would cost $4.00. 7\" -->  \n",
      "<figcaption>  \n",
      "Figure 6: The demo case shows Phi-3-Vision's capability in natural image understanding and reasoning.  \n",
      "</figcaption>  \n",
      "</figure>\u001b[0m\n",
      "\n",
      "***** Chunk 10 *****\n",
      "Length of chunk = 2866\n",
      "\n",
      "\u001b[1;34mArchitecture The Phi-3-Vision (4.2B parameters) is a multimodal model designed to process an image and a textual prompt as inputs, and subsequently generate textual outputs. This model is com- posed of two primary components: an image encoder, i.e., CLIP ViT-L/14 [RKH+21] and a transformer decoder, i.e., phi-3-mini-128K-instruct. The visual tokens, once extracted by the image encoder, are then combined with text tokens in an interleaved way (no particular order for image and text tokens). To ac- commodate high-resolution images and various aspect ratios, a dynamic cropping strategy [DZZ+24b] is utilized to split the input image into a 2d array of blocks, where the tokens of the blocks are concatenated to represent the whole image.  \n",
      "Pre-training The Phi-3-Vision model undergoes a pre-training phase using a diverse dataset, which consists of a combination of interleaved image-text documents (e.g., [LST+24]), image-text pairs from FLD-5B [XWX+24], synthetic data derived from Optical Character Recognition (OCR) of PDF files, datasets for chart/table comprehension, and text-only data. The objective of predicting the next token is employed specifically on text tokens, while any loss associated with image tokens is disregarded during this phase. The pre-training process involves a total of 0.5T tokens that encompass both visual and text elements. During the pre-training phase, the maximum image resolution is capped at 1344 x 1344 as the majority of the training images are smaller than this resolution.  \n",
      "<!-- PageNumber=\"9\" -->  \n",
      "| \\| Coffee 1 - ||| \\| Small \\| Medium \\| Large   |\n",
      "| - | - | - | - |\n",
      "| \\| Espresso | \\| 1.30| \\| 1.60  | 2.00 |\n",
      "| \\| Americana | \\| 1.40 | \\| 1.75| \\| 2.10  |\n",
      "| \\| Cappuccino | \\| 2.25 | \\| 2.90| \\| 3.25  |\n",
      "| \\| Latte | \\| 2.35 | \\| 3.00| \\| 3.25  |\n",
      "| \\| Mocha | \\| 2.35 | \\| 3.00| \\| 3.50  |\n",
      "| \\| Chai | \\| 2.25 | \\| 2.40 | \\| 3.25 |\n",
      "| \\| Chai Bomb | \\| 2.75 | \\| 3.25| \\| 3.75  |\n",
      "| \\| House Cafe | \\| 1.50 | \\| 1.75| \\| 2.00  |\n",
      "| \\| Cafe au Lait | \\| 1.75 | \\| 2.50| \\| 3.25  |\n",
      ":selected:\n",
      "Post-training. The Phi-3-Vision model contains two post-training stages: supervised finetuning (SFT) and direct preference optimization (DPO). For SFT, we leveraged text SFT dataset, public mul- timodal instruct tuning datasets along with large-scale multimodal instruct tuning datasets that we built ourselves, covering diverse domains and tasks such as general natural image understanding, chart/table/- diagram understanding/reasoning, PowerPoint understanding, and model safety. The multimodal SFT data has about a total of 15B tokens. For DPO we mainly use a text DPO dataset and a relatively smaller-scale multimodal DPO dataset. For these two stages, we jointly train multimodal tasks and text- only tasks so that the model can achieve multi-modal reasoning while maintaining language capabilities as much as possible.\u001b[0m\n",
      "\n",
      "***** Chunk 11 *****\n",
      "Length of chunk = 1749\n",
      "\n",
      "\u001b[1;34mWe report in Table 2 the evaluation results of Phi-3-Vision on nine open-source academic benchmarks. These benchmarks evaluate reasoning and perceptual capabilities on visual and text inputs and can be grouped in three categories: Science, Charts, and Generic knowledge. We compare Phi-3-Vision with the following baselines: MM1-3B-Chat [MGF+24], MM1-7B-Chat [MGF+24], Llava-1.6 Vicuna 7B [LLLL23], Llava-1.6 Llama3-8B [LLL+24], Qwen-VL-Chat [BBY+23], Claude 3 Haiku [Ant24], Gemini 1.0 Pro V [TAB+23], and GPT-4V-Turbo. Our performance quality assessment setup used the same evaluation pipeline for all the baselines to ensure a fair comparison, with the exception of MM1-3B-Chat. We just copied and pasted their published numbers since the model is not publicly available.  \n",
      "Our evaluation setup aimed to mimic scenarios where regular users interact with a multi-modal model, i.e., users who are not experts in prompt engineering or know special techniques that can improve performance. For this reason, we adopted the evaluation setting used in Llava-1.5 [LLLL23]. In this setup, the prompts include instructions to select a single letter corresponding to an answer from a list of given options, or answer with a single word or phrase. In our prompts, we did not use specific tokens for multiple-choice questions. Moreover, we did not scale or pre-process any image in our benchmarking system. We placed the images as the first item in the prompts, except on the MMMU dataset where the prompts interleave the images anywhere in the question or the answers. Lastly, our evaluation setup only considered a 0-shot format. Because of these evaluation parameters, our reported numbers can differ from the published numbers of the considered baselines.\u001b[0m\n",
      "\n",
      "***** Chunk 12 *****\n",
      "Length of chunk = 3014\n",
      "\n",
      "\u001b[1;34mTo ensure the integration of Phi-3-Vision aligns with Microsoft's Responsible AI (RAI) principles, we involved safety post-training in both Supervised Fine-Tuning (SFT) stage and Direct Preference Optimization (DPO) stage. In creating the safety training datasets, we utilized not only the text- only RAI datasets, but also a variety of in-house Multi-Modal (MM) RAI datasets that cover various harm categories identified in both public and internal MM RAI benchmarks. For the purpose of RAI evaluation, we performed a rigorous quantitative assessment on both public and internal benchmarks, this was done in conjunction with a human evaluation conducted by Microsoft's internal red team.  \n",
      "In Table 3, we present the evaluation outcomes of Phi-3-Vision on three MM RAI benchmarks: one internal and two public benchmarks (specifically, RTVLM [LLY+24] and VLGuard [ZBY+24]). We juxtapose these results with those of other open-source models such as Llava-1.5 [LLLL23], Llava-1.6 [LLL+24], Qwen-VL-Chat [BBY+23], and GPT4-V[Ope23]. The results clearly indicate that safety post- training notably enhances the RAI performance of Phi-3-Vision across all RAI benchmarks. In Figure 7, we further breakdown the performance across different RAI categories of the VLGuard and Internal benchmarks, demonstrating that safety post-training can aid Phi-3-Vision in improving RAI performance in nearly all categories.  \n",
      "<!-- PageNumber=\"10\" -->  \n",
      "| | 4.2b Phi-3-Vision | MM1-3B-Chat 3.6b [MGF+24] | 7.6b [MGF+24] MM1-7B-Chat | LLaVA-1.6 Vicuna-7b [LLLL23] | LLaVA-Next LLama3-8b [LLL+24] | 9.6b [BBY+23] Qwen-VL-Chat | [Ant24] Claude 3 haiku | [TAB+23] Gemini 1.0 Pro V | GPT-4V-Turbo turbo-2024-04-09 |\n",
      "| - | - | - | - | - | - | - | - | - | - |\n",
      "| MMMU (val) [YNZ+23] | 40.4 | 33.9 | 37.0 | 34.2 | 36.4 | 39.0 | 40.7 | 42.0 | 55.5 |\n",
      "| ScienceQA (test) [LMX+22] | 90.8 | 69.4 | 72.6 | 70.6 | 73.7 | 67.2 | 72.0 | 79.7 | 75.7 |\n",
      "| Math Vista (testmini) [LBX+24] Inter-GPS (test) [LGJ+21] | 44.5 | 32.0 | 35.9 | 31.5 | 34.8 | 29.4 | 33.2 | 35.0 | 47.5 |\n",
      "|| 38.1 | - | - | 20.5 | 24.6 | 22.3 | 32.1 | 28.6 | 41.0 |\n",
      "| MMBench (dev-en) [LDZ+24] POPE (test) [LDZ+23] | 80.5 | 75.9 | 79.0 | 76.3 | 79.4 | 75.8 | 62.4 | 80.0 | 86.1 |\n",
      "|| 85.8 | 87.4 | 86.6 | 87.2 | 87.0 | 82.6 | 74.4 | 84.2 | 83.7 |\n",
      "| AI2D (test) [KSK+16] ChartQA (test) [MLT+22] | 76.7 | - | - | 63.1 | 66.9 | 59.8 | 60.3 | 62.8 | 74.7 |\n",
      "|| 81.4 | - | - | 55.0 | 65.8 | 50.9 | 59.3 | 58.0 | 62.3 |\n",
      "| Text VQA (test) [SNS+19] | 70.9 | 71.9 | 72.8 | 64.6 | 55.7 | 59.4 | 62.7 | 64.7 | 68.1 |  \n",
      "Table 2: Comparison results on public MLLM benchmarks. All the reported numbers are produced with the exact same pipeline to ensure that the numbers are comparable except for MM1-3B-Chat [MGF+24] and MM1-7B- Chat [MGF+24], which are not publicly available. We adopted the evaluation setting used in Llava-1.5 [LLLL23], without any specific prompt or pre-processing image for all results. These numbers might differ from other published numbers due to slightly different prompts.\u001b[0m\n",
      "\n",
      "***** Chunk 13 *****\n",
      "Length of chunk = 2327\n",
      "\n",
      "\u001b[1;34mRegarding the multi-modal LLM capabilities of our Phi-3-Vision, it performs admirably across various fields. However, we have identified certain limitations, particularly with questions necessitating high- level reasoning abilities. Additionally, the model has been observed to occasionally generate ungrounded outputs, making it potentially unreliable in sensitive areas, such as finance. To mitigate these issues, we will incorporate more reasoning-focused and hallucination-related DPO data into post-training in the future.  \n",
      "From a responsible AI standpoint, whilst safety post-training has made significant strides, our Phi- 3-Vision occasionally fails to refrain from answering harmful or sensitive inquiries. Examples of such occasions include deciphering particular types of captcha and describing scam images containing disin- formation or hallucination. We find that this issue partly arises from the capabilities, such as OCR, acquired during the training process with normal instruct tuning datasets, which can be regarded as the trade-off between helpfulness and harmlessness. Moving forward, we need to further explore this area to achieve a better balance.  \n",
      "<!-- PageNumber=\"11\" -->  \n",
      "| |||| Phi-3-Vision Phi-3-Vision w/o safety Llava-1.6 Vicuna Qwen-VL-Chat 3.8b+0.3b  3.8b+0.3b  7b+0.3b  7.7b+1.9b | GPT4-V N/A |\n",
      "| - | - | - | - | - | - |\n",
      "| Internal (private) | 8.30 | 7.06 | 5.44 | 7.27 | 8.55 |\n",
      "| RTVLM (public) | 4.64 | 3.56 | 3.86 | 4.78 | 6.81 |\n",
      "| VLGuard (public) | 9.12 | 4.66 | 5.62 | 8.33 | 8.90 |  \n",
      "Table 3: Comparison results on public and private multi-modal RAI benchmarks. Note that all metrics in the table are [0,10] and a higher value indicates a better performance.  \n",
      "<figure>  \n",
      "![](figures/6)  \n",
      "<!-- FigureContent=\"Phi-3-Vision w/o safety Phi-3-Vision Phi-3-Vision w/o safety Phi-3-Vision 10 10 8 8 6 6 4 4 2 2 0 0 safeimg-safetxt deception discrimination privacy risk-behaviour safeimg-unsafetxt Total harmful jailbreak privacy grounding Total\" -->  \n",
      "<figcaption>  \n",
      "Figure 7: Comparison of categorized RAI performance of Phi-3-Vision with and without the safety post-training\n",
      "on the VLGuard (left) and Internal (right) benchmark, respectively. It clearly indicates that safety post-training\n",
      "can enhance the RAI performance across nearly all the RAI categories.  \n",
      "</figcaption>  \n",
      "</figure>\u001b[0m\n",
      "\n",
      "***** Chunk 14 *****\n",
      "Length of chunk = 16600\n",
      "\n",
      "\u001b[1;34m[AI] Meta AI. Introducing meta llama 3: The most capable openly available llm to date.  \n",
      "[Ant24] AI Anthropic. The claude 3 model family: Opus, sonnet, haiku. Claude-3 Model Card, 2024.  \n",
      "[AON+21] Jacob Austin, Augustus Odena, Maxwell Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie Cai, Michael Terry, Quoc Le, and Charles Sutton. Program synthesis with large language models. arXiv preprint ar Xiv:2108.07732, 2021.  \n",
      "[BBY+23] Jinze Bai, Shuai Bai, Shusheng Yang, Shijie Wang, Sinan Tan, Peng Wang, Junyang Lin, Chang Zhou, and Jingren Zhou. Qwen-vl: A versatile vision-language model for understand- ing, localization, text reading, and beyond, 2023.  \n",
      "[BJN+22] Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, Nicholas Joseph, Saurav Kada- vath, Jackson Kernion, Tom Conerly, Sheer El-Showk, Nelson Elhage, Zac Hatfield-Dodds, Danny Hernandez, Tristan Hume, Scott Johnston, Shauna Kravec, Liane Lovitt, Neel Nanda, Catherine Olsson, Dario Amodei, Tom Brown, Jack Clark, Sam McCandlish, Chris Olah, Ben Mann, and Jared Kaplan. Training a helpful and harmless assistant with reinforcement learning from human feedback, 2022.  \n",
      "<!-- PageNumber=\"12\" -->  \n",
      "[BSA+24] Federico Bianchi, Mirac Suzgun, Giuseppe Attanasio, Paul Röttger, Dan Jurafsky, Tatsunori Hashimoto, and James Zou. Safety-tuned llamas: Lessons from improving the safety of large language models that follow instructions, 2024.  \n",
      "[BZGC19] Yonatan Bisk, Rowan Zellers, Jianfeng Gao, and Yejin Choi. Piqa: Reasoning about physical commonsense in natural language. arXiv preprint arXiv:1911.11641, 2019.  \n",
      "[CCE+18] Peter Clark, Isaac Cowhey, Oren Etzioni, Tushar Khot, Ashish Sabharwal, Carissa Schoenick, and Oyvind Tafjord. Think you have solved question answering? try arc, the ai2 reasoning challenge, 2018.  \n",
      "[CKB+21] Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, Christopher Hesse, and John Schulman. Training verifiers to solve math word problems. ar Xiv preprint ar Xiv:2110.14168, 2021.  \n",
      "[CLC+19] Christopher Clark, Kenton Lee, Ming-Wei Chang, Tom Kwiatkowski, Michael Collins, and Kristina Toutanova. Boolq: Exploring the surprising difficulty of natural yes/no questions. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pages 2924-2936, 2019.  \n",
      "[CTJ+21] Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mo- hammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, and Wojciech Zaremba. Evaluating large language models trained on code, 2021.  \n",
      "[DFE+22] Tri Dao, Dan Fu, Stefano Ermon, Atri Rudra, and Christopher Re. Flashattention: Fast and memory-efficient exact attention with io-awareness. Advances in Neural Information Processing Systems, 35:16344-16359, 2022.  \n",
      "[DZZ+24a] Yiran Ding, Li Lyna Zhang, Chengruidong Zhang, Yuanyuan Xu, Ning Shang, Jiahang Xu, Fan Yang, and Mao Yang. Longrope: Extending llm context window beyond 2 million tokens, 2024.  \n",
      "[DZZ+24b] Xiaoyi Dong, Pan Zhang, Yuhang Zang, Yuhang Cao, Bin Wang, Linke Ouyang, Songyang Zhang, Haodong Duan, Wenwei Zhang, Yining Li, et al. Internlm-xcomposer2-4khd: A pioneering large vision-language model handling resolutions from 336 pixels to 4k hd. arXiv preprint ar Xiv:2404.06512, 2024.  \n",
      "[GZA+23] Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Gustavo de Rosa Piero Kauffmann, Olli Saarikivia,  \n",
      "<!-- PageNumber=\"13\" -->  \n",
      "Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen El- dan, Adam Tauman Kalai, Yin Tat Lee, and Yuanzhi Li. Textbooks are all you need. arXiv preprint ar Xiv:2306.11644, 2023.  \n",
      "[HBK+21] Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, and Jacob Steinhardt. Measuring mathematical problem solving with the MATH dataset, 2021.  \n",
      "[HBM+22] Jordan Hoffmann, Sebastian Borgeaud, Arthur Mensch, Elena Buchatskaya, Eliza Ruther- ford Trevor Cai, Diego de Las Casas, Lisa Anne Hendricks, Johannes Welbl, Aidan Clark, Tom Hennigan, Eric Noland, Katie Millican, George van den Driessche, Bogdan Damoc, Au- relia Guy, Simon Osindero, Karen Simonyan, Erich Elsen, Jack W. Rae, Oriol Vinyals, and Laurent Sifre. Training compute-optimal large language models. arXiv preprint ar Xiv:2203.15556, 2022.  \n",
      "[JBA+23] Mojan Javaheripi, Sebastien Bubeck, Marah Abdin, Jyoti Aneja, Caio Cesar Teodoro Mendes, Weizhu Chen, Allie Del Giorno, Ronen Eldan, Sivakanth Gopi, Suriya Gunasekar, Piero Kauffmann, Yin Tat Lee, Yuanzhi Li, Anh Nguyen, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Michael Santacroce, Harkirat Singh Behl, Adam Tau- mann Kalai, Xin Wang, Rachel Ward, Philipp Witte, Cyril Zhang, and Yi Zhang. Phi-2: The surprising power of small language models. Microsoft Research Blog, 2023.  \n",
      "[JCWZ17] Mandar Joshi, Eunsol Choi, Daniel S. Weld, and Luke Zettlemoyer. Triviaqa: A large scale distantly supervised challenge dataset for reading comprehension, 2017.  \n",
      "[JLD+23] Jiaming Ji, Mickel Liu, Juntao Dai, Xuehai Pan, Chi Zhang, Ce Bian, Chi Zhang, Ruiyang Sun, Yizhou Wang, and Yaodong Yang. Beavertails: Towards improved safety alignment of llm via a human-preference dataset, 2023.  \n",
      "[JPO+20] Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits. What disease does this patient have? a large-scale open domain question answering dataset from medical exams, 2020.  \n",
      "[JSM+23] Albert Q. Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, Lélio Renard Lavaud, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William El Sayed. Mistral 7b, 2023.  \n",
      "[JSR+24] Albert Q. Jiang, Alexandre Sablayrolles, Antoine Roux, Arthur Mensch, Blanche Savary, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Emma Bou Hanna, Florian Bressand, Gianna Lengyel, Guillaume Bour, Guillaume Lample, Lélio Renard Lavaud, Lu- cile Saulnier, Marie-Anne Lachaux, Pierre Stock, Sandeep Subramanian, Sophia Yang, Szy- mon Antoniak, Teven Le Scao, Théophile Gervet, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William El Sayed. Mixtral of experts, 2024.  \n",
      "[KLZ+23] Woosuk Kwon, Zhuohan Li, Siyuan Zhuang, Ying Sheng, Lianmin Zheng, Cody Hao Yu, Joseph E. Gonzalez, Hao Zhang, and Ion Stoica. Efficient memory management for large language model serving with pagedattention. In Proceedings of the ACM SIGOPS 29th Symposium on Operating Systems Principles, 2023.  \n",
      "<!-- PageNumber=\"14\" -->  \n",
      "[KMH+20] Jared Kaplan, Sam McCandlish, Tom Henighan, Tom B Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. Scaling laws for neural language models. arXiv preprint ar Xiv:2001.08361, 2020.  \n",
      "[KSK+16] Aniruddha Kembhavi, Mike Salvato, Eric Kolve, Minjoon Seo, Hannaneh Hajishirzi, and Ali Farhadi. A diagram is worth a dozen images, 2016.  \n",
      "[LBE+23] Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar, and Yin Tat Lee. Textbooks are all you need ii: phi-1.5 technical report. arXiv preprint ar Xiv:2309.05463, 2023.  \n",
      "[LBX+24] Pan Lu, Hritik Bansal, Tony Xia, Jiacheng Liu, Chunyuan Li, Hannaneh Hajishirzi, Hao Cheng, Kai-Wei Chang, Michel Galley, and Jianfeng Gao. Mathvista: Evaluating mathe- matical reasoning of foundation models in visual contexts, 2024.  \n",
      "[LDZ+23] Yifan Li, Yifan Du, Kun Zhou, Jinpeng Wang, Wayne Xin Zhao, and Ji-Rong Wen. Evalu- ating object hallucination in large vision-language models, 2023.  \n",
      "[LDZ+24] Yuan Liu, Haodong Duan, Yuanhan Zhang, Bo Li, Songyang Zhang, Wangbo Zhao, Yike Yuan, Jiaqi Wang, Conghui He, Ziwei Liu, Kai Chen, and Dahua Lin. Mmbench: Is your multi-modal model an all-around player?, 2024.  \n",
      "[LGJ+21] Pan Lu, Ran Gong, Shibiao Jiang, Liang Qiu, Siyuan Huang, Xiaodan Liang, and Song-Chun Zhu. Inter-gps: Interpretable geometry problem solving with formal language and symbolic reasoning, 2021.  \n",
      "[LHE22] Stephanie Lin, Jacob Hilton, and Owain Evans. Truthfulqa: Measuring how models mimic human falsehoods, 2022.  \n",
      "[LLL+24] Haotian Liu, Chunyuan Li, Yuheng Li, Bo Li, Yuanhan Zhang, Sheng Shen, and Yong Jae Lee. Llava-next: Improved reasoning, ocr, and world knowledge, January 2024.  \n",
      "[LLLL23] Haotian Liu, Chunyuan Li, Yuheng Li, and Yong Jae Lee. Improved baselines with visual instruction tuning. arXiv preprint ar Xiv:2310.03744, 2023.  \n",
      "[LLY+24] Mukai Li, Lei Li, Yuwei Yin, Masood Ahmed, Zhenguang Liu, and Qi Liu. Red teaming visual language models. ar Xiv preprint ar Xiv:2401.12915, 2024.  \n",
      "[LMX+22] Pan Lu, Swaroop Mishra, Tony Xia, Liang Qiu, Kai-Wei Chang, Song-Chun Zhu, Oyvind Tafjord, Peter Clark, and Ashwin Kalyan. Learn to explain: Multimodal reasoning via thought chains for science question answering. In The 36th Conference on Neural Information Processing Systems (NeurIPS), 2022.  \n",
      "[LST+24] Hugo Laurencon, Lucile Saulnier, Leo Tronchon, Stas Bekman, Amanpreet Singh, An- ton Lozhkov, Thomas Wang, Siddharth Karamcheti, Alexander Rush, Douwe Kiela, et al. Obelics: An open web-scale filtered dataset of interleaved image-text documents. Advances in Neural Information Processing Systems, 36, 2024.  \n",
      "[MCKS18] Todor Mihaylov, Peter Clark, Tushar Khot, and Ashish Sabharwal. Can a suit of armor conduct electricity? a new dataset for open book question answering, 2018.  \n",
      "<!-- PageNumber=\"15\" -->  \n",
      "[MGF+24] Brandon McKinzie, Zhe Gan, Jean-Philippe Fauconnier, Sam Dodge, Bowen Zhang, Philipp Dufter, Dhruti Shah, Xianzhi Du, Futang Peng, Floris Weers, Anton Belyi, Haotian Zhang, Karanjeet Singh, Doug Kang, Ankur Jain, Hongyu Hè, Max Schwarzer, Tom Gunter, Xiang Kong, Aonan Zhang, Jianyu Wang, Chong Wang, Nan Du, Tao Lei, Sam Wiseman, Guoli Yin, Mark Lee, Zirui Wang, Ruoming Pang, Peter Grasch, Alexander Toshev, and Yinfei Yang. Mm1: Methods, analysis & insights from multimodal llm pre-training. ar Xiv preprint ar Xiv:2403.09611, 2024.  \n",
      "[MHJ+23] Ahmed Magooda, Alec Helyar, Kyle Jackson, David Sullivan, Chad Atalla, Emily Sheng, Dan Vann, Richard Edgar, Hamid Palangi, Roman Lutz, Hongliang Kong, Vincent Yun, Eslam Kamal, Federico Zarfati, Hanna Wallach, Sarah Bird, and Mei Chen. A framework for automated measurement of responsible ai harms in generative ai applications, 2023.  \n",
      "[MLT+22] Ahmed Masry, Do Long, Jia Qing Tan, Shafiq Joty, and Enamul Hoque. ChartQA: A benchmark for question answering about charts with visual and logical reasoning. In Findings of the Association for Computational Linguistics: ACL 2022, pages 2263-2279, Dublin, Ireland, May 2022. Association for Computational Linguistics.  \n",
      "[MRB+23] Niklas Muennighoff, Alexander M Rush, Boaz Barak, Teven Le Scao, Aleksandra Piktus, Nouamane Tazi, Sampo Pyysalo, Thomas Wolf, and Colin Raffel. Scaling data-constrained language models. arXiv preprint arXiv:2305.16264, 2023.  \n",
      "[NWD+20] Yixin Nie, Adina Williams, Emily Dinan, Mohit Bansal, Jason Weston, and Douwe Kiela. Adversarial nli: A new benchmark for natural language understanding, 2020.  \n",
      "[Ope23] OpenAI. Gpt-4v(ision) system card, 2023. https://cdn.openai.com/papers/GPTV\\_ System\\_Card. pdf.  \n",
      "[RHS+23] David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, and Samuel R. Bowman. Gpqa: A graduate-level google-proof q&a benchmark, 2023.  \n",
      "[RKH+21] Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agar- wal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et al. Learning transferable visual models from natural language supervision. In International conference on machine learning, pages 8748-8763. PMLR, 2021.  \n",
      "[RWC+19] Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, and Ilya Sutskever. Language models are unsupervised multitask learners. OpenAI blog, 1(8):9, 2019.  \n",
      "[SLBBC19] Keisuke Sakaguchi, Ronan Le Bras, Chandra Bhagavatula, and Yejin Choi. Winogrande: An adversarial winograd schema challenge at scale. arXiv preprint arXiv:1907.10641, 2019.  \n",
      "[SNS+19] Amanpreet Singh, Vivek Natarajan, Meet Shah, Yu Jiang, Xinlei Chen, Dhruv Batra, Devi Parikh, and Marcus Rohrbach. Towards vqa models that can read, 2019.  \n",
      "[SRR+22] Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adrià Garriga-Alonso, et al. Beyond the imitation game: Quantifying and extrapolating the capabilities of language models. arXiv preprint ar Xiv:2206.04615, 2022.  \n",
      "<!-- PageNumber=\"16\" -->  \n",
      "[SSS+22] Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, and Jason Wei. Chal- lenging big-bench tasks and whether chain-of-thought can solve them, 2022.  \n",
      "[TAB+23] Gemini Team, Rohan Anil, Sebastian Borgeaud, Yonghui Wu, Jean-Baptiste Alayrac, Jiahui Yu, Radu Soricut, Johan Schalkwyk, Andrew M Dai, Anja Hauth, et al. Gemini: a family of highly capable multimodal models. arXiv preprint ar Xiv:2312.11805, 2023.  \n",
      "[THLB19] Alon Talmor, Jonathan Herzig, Nicholas Lourie, and Jonathan Berant. Commonsenseqa: A question answering challenge targeting commonsense knowledge, 2019.  \n",
      "[TLI+23] Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, and Guillaume Lample. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971, 2023.  \n",
      "[TMH+24] Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivière, Mihir Sanjay Kale, Juliette Love, et al. Gemma: Open models based on gemini research and technology, 2024.  \n",
      "[VSP+17] Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, L ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In Advances in Neural Information Processing Systems, volume 30, 2017.  \n",
      "[XWX+24] Bin Xiao, Haiping Wu, Weijian Xu, Xiyang Dai, Houdong Hu, Yumao Lu, Michael Zeng, Ce Liu, and Lu Yuan. Florence-2: Advancing a unified representation for a variety of vision tasks. 2024.  \n",
      "[YNZ+23] Xiang Yue, Yuansheng Ni, Kai Zhang, Tianyu Zheng, Ruoqi Liu, Ge Zhang, Samuel Stevens, Dongfu Jiang, Weiming Ren, Yuxuan Sun, Cong Wei, Botao Yu, Ruibin Yuan, Renliang Sun, Ming Yin, Boyuan Zheng, Zhenzhu Yang, Yibo Liu, Wenhao Huang, Huan Sun, Yu Su, and Wenhu Chen. Mmmu: A massive multi-discipline multimodal understanding and reasoning benchmark for expert agi, 2023.  \n",
      "[ZBY+24] Yongshuo Zong, Ondrej Bohdal, Tingyang Yu, Yongxin Yang, and Timothy Hospedales. Safety fine-tuning at (almost) no cost: A baseline for vision large language models. arXiv preprint ar Xiv:2402.02207, 2024.  \n",
      "[ZCG+23] Wanjun Zhong, Ruixiang Cui, Yiduo Guo, Yaobo Liang, Shuai Lu, Yanlin Wang, Amin Saied, Weizhu Chen, and Nan Duan. Agieval: A human-centric benchmark for evaluating foundation models, 2023.  \n",
      "[ZCS+23] Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi Lin, Zhuohan Li, Dacheng Li, Eric Xing, et al. Judging llm-as-a-judge with mt-bench and chatbot arena. arXiv preprint ar Xiv:2306.05685, 2023.  \n",
      "[ZHB+19] Rowan Zellers, Ari Holtzman, Yonatan Bisk, Ali Farhadi, and Yejin Choi. Hellaswag: Can a machine really finish your sentence? In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pages 4791-4800, 2019.  \n",
      "<!-- PageNumber=\"17\" -->\u001b[0m\n",
      "\n",
      "***** Chunk 15 *****\n",
      "Length of chunk = 441\n",
      "\n",
      "\u001b[1;34mQuestion:  \n",
      "Solve for x: (-3)(-4-3x)=2  \n",
      "Options :  \n",
      "A. - 5 6  \n",
      "B. 7  \n",
      "6  \n",
      "C. 5  \n",
      "3  \n",
      "D.  \n",
      "1  \n",
      "Answer: A  \n",
      "Question: Which of the following is the body cavity that contains the pituitary gland? Options :  \n",
      "A. Abdominal  \n",
      "B. Cranial  \n",
      "C. Pleural  \n",
      "D. Spinal  \n",
      "Answer: B  \n",
      "Question: Where was the most famous site of the mystery cults in Greece? Options :  \n",
      "A. Ephesus  \n",
      "B. Corinth  \n",
      "C. Athens  \n",
      "D. Eleusis  \n",
      "Answer:  \n",
      "<!-- PageNumber=\"18\" -->\u001b[0m\n",
      "\n",
      "***** Chunk 16 *****\n",
      "Length of chunk = 1851\n",
      "\n",
      "\u001b[1;34m||||\n",
      "| - | - | - |\n",
      "| Marah Abdin | Jamie Huynh | Olli Saarikivi |\n",
      "| Sam Ade Jacobs | Dan Iter | Amin Saied |\n",
      "| Ammar Ahmad Awan | Russell J. Hewett | Adil Salim |\n",
      "| Jyoti Aneja | Mojan Javaheripi | Michael Santacroce |\n",
      "| Hany Awadalla | Xin Jin | Shital Shah |\n",
      "| Ahmed Awadallah | Nikos Karampatziakis | Ning Shang |\n",
      "| Nguyen Bach | Piero Kauffmann | Hiteshi Sharma |\n",
      "| Amit Bahree | Mahoud Khademi | Swadheen Shukla |\n",
      "| Arash Bakhtiari | Dongwoo Kim | Xia Song |\n",
      "| Jianmin Bao | Lev Kurilenko | Yin Tat Lee |\n",
      "| Harkirat Behl | Yuanzhi Li | Andrea Tupini |\n",
      "| Alon Benhaim | Yunsheng Li | Xin Wang |\n",
      "| Misha Bilenko | Chen Liang | Guanhua Wang |\n",
      "| Johan Bjorck | Lars Liden | Lijuan Wang |\n",
      "| Sébastien Bubeck | Eric Lin | Chunyu Wang :unselected: |\n",
      "| Caio César Teodoro Mendes | Zeqi Lin | Yu Wang |\n",
      "| Martin Cai | Weishung Liu | Rachel Ward |\n",
      "| Qin Cai | Mengchen Liu | Wen Wen |\n",
      "| Vishrav Chaudhary | Ce Liu | Philipp Witte |\n",
      "| Weizhu Chen | Chong Luo :unselected: | Haiping Wu |\n",
      "| Yi-Ling Chen | Li Lyna Zhang | Michael Wyatt |\n",
      "| Yen-Chun Chen | Piyush Madan | Bin Xiao |\n",
      "| Dongdong Chen | David Majercak | Jiahang Xu |\n",
      "| Dong Chen | Matt Mazzola | Can Xu |\n",
      "| Parul Chopra | Arindam Mitra | Weijian Xu |\n",
      "| Xiyang Dai | Hardik Modi | Sonali Yadav |\n",
      "| Gustavo de Rosa | Anh Nguyen | Fan Yang |\n",
      "| Allie Del Giorno | Brandon Norick | Ziyi Yang |\n",
      "| Matthew Dixon | Barun Patra | Jianwei Yang |\n",
      "| Ronen Eldan | Daniel Perez-Becker | Yifan Yang |\n",
      "| Victor Fragoso | Thomas Portet | Donghan Yu :unselected: |\n",
      "| Mei Gao | Reid Pryzant | Lu Yuan |\n",
      "| Min Gao | Heyang Qin | Chengruidong Zhang :unselected: |\n",
      "| Jianfeng Gao | James R. Lee | Cyril Zhang :unselected: |\n",
      "| Abhishek Goswami | Marko Radmilac | Jianwen Zhang |\n",
      "| Suriya Gunasekar | Corby Rosset | Yi Zhang |\n",
      "| Emman Haider | Sambudha Roy | Yunan Zhang |\n",
      "| Junheng Hao | Olatunji Ruwase | Xiren Zhou |\u001b[0m\n",
      "\n",
      "***** Chunk 17 *****\n",
      "Length of chunk = 164\n",
      "\n",
      "\u001b[1;34mWe would like to thank Zhuohan Li, Simon Mo, and Kaichao You from UC Berkeley for sharing their insights on the vLLM kernel.  \n",
      "<!-- PageNumber=\"19\" -->\n",
      ":unselected:\u001b[0m\n",
      "\n"
     ]
    }
   ],
   "source": [
    "chunk_size_list = []\n",
    "\n",
    "for idx, chunk in enumerate(chunks, start=1):\n",
    "    print(f\"***** Chunk {idx} *****\\nLength of chunk = {len(chunk.page_content)}\\n\")\n",
    "    print(f\"\\033[1;34m{chunk.page_content}\\033[0m\\n\")\n",
    "    chunk_size_list.append(len(chunk.page_content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9c954316",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABOcAAAJwCAYAAADC0QVwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAADGa0lEQVR4nOzdeXiU9b3+8Xuyb8wkIRshIQlhC/sSoLhbKIhWi3X5aam1lsqpdSlVqdoFjLZa0bprFU/derS1ttVaa1Xqhi0IhBCBEFYDJISEhCQzJCH7/P7gZI4pqJDy5DNw36/r8rpKZpj5vOf79Jnky2TG5ff7/RAREREREREREZE+F2I9gIiIiIiIiIiICCttzomIiIiIiIiIiBjR5pyIiIiIiIiIiIgRbc6JiIiIiIiIiIgY0eaciIiIiIiIiIiIEW3OiYiIiIiIiIiIGNHmnIiIiIiIiIiIiBFtzomIiIiIiIiIiBjR5pyIiIiIiIiIiIgRbc6JiIgIlezsbHz729+2HuOIOjo68KMf/QiZmZkICQnBnDlzrEcKWt/+9reRnZ3dp/cXFxfXZ/f3aTt37oTL5cJ9991ncv8iIiLiLG3OiYiIyElhw4YNuPjii5GVlYWoqCgMHDgQX/nKV/DII49Yj3bUnn76adx77724+OKL8dxzz+GHP/zhZ163q6sLzz//PKZOnYrExET069cPw4YNw7e+9S189NFHfTi1cyorK3H77bejuLjYehQRERERx4RZDyAiIiLyn1qxYgXOPvtsDBo0CFdffTXS0tJQXl6Ojz76CA899BCuv/76wHW3bNmCkJDg/PfJd999FwMHDsQDDzzwhde94YYb8Nhjj+FrX/sa5s6di7CwMGzZsgV///vfMXjwYHzpS1/qg4mdVVlZiYKCAmRnZ2P8+PE9LnvqqafQ1dVlM5iIiIjIcaTNORERETnh/eIXv4DH48GaNWsQHx/f47J9+/b1+HNkZGQfTnZs9u3bd9j8R1JdXY3HH38cV199NZYuXdrjsgcffBA1NTUOTRg8wsPDrUcQEREROS6C85+NRURERI7Bjh07MGrUqCNubKWkpPT487+/55zL5frM/3bu3Bm43ubNm3HxxRcjMTERUVFRyM/Px2uvvXZU8zU1NeGmm25CZmYmIiMjMXz4cNx3333w+/0A/u89xd577z2UlJQE7v/9998/4u2VlZXB7/fj1FNPPewyl8t1WHNDQwMWLFgQuP8hQ4bgnnvu6fHKs0+/r9ljjz2GwYMHIyYmBjNnzkR5eTn8fj/uvPNOZGRkIDo6Gl/72tdQV1fX437+8pe/4LzzzkN6ejoiIyORm5uLO++8E52dnT2ud9ZZZ2H06NHYtGkTzj77bMTExGDgwIFYsmRJ4Drvv/8+Jk+eDAC46qqrAo/Js88+C+DI7znX1dWFhx56CGPGjEFUVBSSk5NxzjnnoLCw8LMX53+tWrUK5557LhISEhAbG4uxY8fioYceOux6e/bswZw5cxAXF4fk5GTcfPPNPfref//9I65d9+PbPX93Q1xc3Bfe5pH4/X7Mnz8fERER+POf/wwAaG9vR0FBAYYOHYqoqCj0798fp512GpYtW/aF/SIiImJHr5wTERGRE15WVhZWrlyJjRs3YvTo0cf0d3/7298e9rWf/vSn2LdvX+ADAEpKSnDqqadi4MCBuPXWWxEbG4s//OEPmDNnDv70pz/hwgsv/Mzb9/v9uOCCC/Dee+9h3rx5GD9+PN566y0sXLgQe/bswQMPPIDk5GT89re/xS9+8Qs0Njbi7rvvBgDk5eV9Zi8AvPzyy7jkkksQExPzmfff3NyMM888E3v27MF//dd/YdCgQVixYgVuu+027N27Fw8++GCP67/wwgtoa2vD9ddfj7q6OixZsgSXXnopvvzlL+P999/HLbfcgu3bt+ORRx7BzTffjKeffjrwd5999lnExcXhxhtvRFxcHN59910sWrQIPp8P9957b4/7qa+vxznnnIOvf/3ruPTSS/HHP/4Rt9xyC8aMGYPZs2cjLy8Pd9xxBxYtWoT58+fj9NNPBwCccsopn9k6b948PPvss5g9eza++93voqOjAx9++CE++ugj5Ofnf+bfW7ZsGb761a9iwIAB+MEPfoC0tDSUlpbi9ddfxw9+8IPA9To7OzFr1ixMnToV9913H/7xj3/gV7/6FXJzc3HNNdd85u1/nt7cZmdnJ77zne/gpZdewiuvvILzzjsPAHD77bfj7rvvxne/+11MmTIFPp8PhYWFKCoqwle+8pVezSciIiJ9wC8iIiJygnv77bf9oaGh/tDQUP+0adP8P/rRj/xvvfWWv62t7bDrZmVl+a+88srPvK0lS5b4Afiff/75wNemT5/uHzNmjL+lpSXwta6uLv8pp5ziHzp06OfO9uqrr/oB+H/+85/3+PrFF1/sd7lc/u3btwe+duaZZ/pHjRr1Rbl+v9/v/9a3vuUH4E9ISPBfeOGF/vvuu89fWlp62PXuvPNOf2xsrH/r1q09vn7rrbf6Q0ND/bt37/b7/X5/WVmZH4A/OTnZ39DQELjebbfd5gfgHzdunL+9vT3w9csvv9wfERHR4zFpbm4+7P7/67/+yx8TE9PjemeeeeZhj3Fra6s/LS3Nf9FFFwW+tmbNGj8A/zPPPHPY7V555ZX+rKyswJ/fffddPwD/DTfccNh1u7q6Dvtat46ODn9OTo4/KyvLX19f/5l/78orr/QD8N9xxx09rjNhwgT/pEmTAn9+7733/AD87733Xo/rdT++n2452tvs/rv33nuvv7293f///t//80dHR/vfeuutHn9v3Lhx/vPOO+8zW0VERCQ46ddaRURE5IT3la98BStXrsQFF1yAjz/+GEuWLMGsWbMwcODAo/7VUwB47733cNttt+H666/HFVdcAQCoq6vDu+++i0svvRQHDhxAbW0tamtrsX//fsyaNQvbtm3Dnj17PvM233jjDYSGhuKGG27o8fWbbroJfr8ff//733vV/Mwzz+DRRx9FTk4OXnnlFdx8883Iy8vD9OnTe8zz8ssv4/TTT0dCQkJg9traWsyYMQOdnZ1Yvnx5j9u95JJL4PF4An+eOnUqAOCb3/wmwsLCeny9ra2tx31FR0cH/nf3Y3X66aejubkZmzdv7nE/cXFx+OY3vxn4c0REBKZMmYJPPvmkV4/Hn/70J7hcLixevPiwy1wu12f+vXXr1qGsrAwLFiw47Neij/T3vve97/X48+mnn97rmY/1Ntva2nDJJZfg9ddfxxtvvIGZM2f2uDw+Ph4lJSXYtm3bfzSPiIiI9C1tzomIiMhJYfLkyfjzn/+M+vp6rF69GrfddhsOHDiAiy++GJs2bfrCv19RUYH/9//+H0499VTcf//9ga9v374dfr8fP/vZz5CcnNzjv+6NoH//0IlP27VrF9LT09GvX78eX+/+ldVdu3b1JhchISG49tprsXbtWtTW1uIvf/kLZs+ejXfffReXXXZZ4Hrbtm3Dm2++edjsM2bMOOLsgwYN6vHn7o26zMzMI369vr4+8LWSkhJceOGF8Hg8cLvdSE5ODmzAeb3eHn8/IyPjsM2vhISEHrd3LHbs2IH09HQkJiYe898DcFS/Dt39Pnaf9p/MfKy3effdd+PVV1/FH//4R5x11lmHXX7HHXegoaEBw4YNw5gxY7Bw4UKsX7++17OJiIhI39B7zomIiMhJJSIiApMnT8bkyZMxbNgwXHXVVXj55ZeP+Iqqbm1tbbj44osRGRmJP/zhDz1eIdb9oQk333wzZs2adcS/P2TIkOMbcYz69++PCy64ABdccAHOOussfPDBB9i1axeysrLQ1dWFr3zlK/jRj350xL87bNiwHn8ODQ094vU+6+v+//1Qi4aGBpx55plwu9244447kJubi6ioKBQVFeGWW27p8eETR3N7weizZv60z3qV3md9wMPR3Ga3WbNm4c0338SSJUtw1llnISoqqsflZ5xxBnbs2IG//OUvePvtt/Hf//3feOCBB/DEE0/gu9/97lHfj4iIiPQtbc6JiIjISav7QwD27t37ude74YYbUFxcjOXLlyM1NbXHZYMHDwYAhIeHB15tdiyysrLwj3/8AwcOHOjx6rnuX/Ps/nCH4yU/Px8ffPAB9u7di6ysLOTm5qKxsbFXsx+L999/H/v378ef//xnnHHGGYGvl5WV9fo2P+/XUf9dbm4u3nrrLdTV1R3Tq+dyc3MBABs3bjwuj1FCQgKAQ5uVn9bbV0h+2pe+9CV873vfw1e/+lVccskleOWVV3psJANAYmIirrrqKlx11VVobGzEGWecgdtvv12bcyIiIkFMv9YqIiIiJ7z33nvviK+4euONNwAAw4cP/8y/+8wzz+DJJ5/EY489hilTphx2eUpKCs466yw8+eSTR9zkq6mp+dzZzj33XHR2duLRRx/t8fUHHngALpcLs2fP/ty/fyRVVVVH/FXdtrY2vPPOOwgJCQm8mu/SSy/FypUr8dZbbx12/YaGBnR0dBzz/R9J9yvAPr0ObW1tePzxx3t9m7GxsQAO3+g6kosuugh+vx8FBQWHXfZ5r8abOHEicnJy8OCDDx52P715FV9WVhZCQ0MPey+//+Rx+LQZM2bg97//Pd58801cccUVPV6RuH///h7XjYuLw5AhQ9Da2npc7ltEREScoVfOiYiIyAnv+uuvR3NzMy688EKMGDECbW1tWLFiBV566SVkZ2fjqquuOuLfq62txfe//32MHDkSkZGR+J//+Z8el1944YWIjY3FY489htNOOw1jxozB1VdfjcGDB6O6uhorV65ERUUFPv7448+c7fzzz8fZZ5+Nn/zkJ9i5cyfGjRuHt99+G3/5y1+wYMGCwCu3jkVFRQWmTJmCL3/5y5g+fTrS0tKwb98+/O53v8PHH3+MBQsWICkpCQCwcOFCvPbaa/jqV7+Kb3/725g0aRKampqwYcMG/PGPf8TOnTsD1/1PnHLKKUhISMCVV16JG264AS6XC7/97W//o19Tzc3NRXx8PJ544gn069cPsbGxmDp1KnJycg677tlnn40rrrgCDz/8MLZt24ZzzjkHXV1d+PDDD3H22WfjuuuuO+J9hISE4Ne//jXOP/98jB8/HldddRUGDBiAzZs3o6Sk5Iibmp/H4/HgkksuwSOPPAKXy4Xc3Fy8/vrrn/u+hMdqzpw5eOaZZ/Ctb30LbrcbTz75JABg5MiROOusszBp0iQkJiaisLAQf/zjHz+zXURERIKDNudERETkhHfffffh5ZdfxhtvvIGlS5eira0NgwYNwve//3389Kc/PexTOLs1NjaipaUFmzZtCnw666eVlZUhNjYWI0eORGFhIQoKCvDss89i//79SElJwYQJE7Bo0aLPnS0kJASvvfYaFi1ahJdeegnPPPMMsrOzce+99+Kmm27qVe/w4cPx4IMP4o033sDjjz+O6upqREVFYfTo0Xjqqacwb968wHVjYmLwwQcf4K677sLLL7+M559/Hm63G8OGDUNBQUGPT2b9T/Tv3x+vv/46brrpJvz0pz9FQkICvvnNb2L69Omf+V59XyQ8PBzPPfccbrvtNnzve99DR0cHnnnmmSNuzgGHXgU5duxY/OY3v8HChQvh8XiQn5+PU0455XPvZ9asWXjvvfdQUFCAX/3qV+jq6kJubi6uvvrqXs39yCOPoL29HU888QQiIyNx6aWX4t577z2qD504Wt/85jdx4MABfP/734fb7ca9996LG264Aa+99hrefvtttLa2IisrCz//+c+xcOHC43a/IiIicvy5/MH8rrsiIiIiIiIiIiInMb3nnIiIiIiIiIiIiBFtzomIiIiIiIiIiBjR5pyIiIiIiIiIiIgRbc6JiIiIiIiIiIgY0eaciIiIiIiIiIiIEW3OiYiIiIiIiIiIGAmzHuBk0dXVhcrKSvTr1w8ul8t6HBERERERERERMeT3+3HgwAGkp6cjJOSzXx+nzbnjpLKyEpmZmdZjiIiIiIiIiIhIECkvL0dGRsZnXq7NueOkX79+AA494G6323iaE0thYSHy8/OtxzChdrWzUbva2aidr521G1C72vmoXe1smNt7y+fzITMzM7Bn9Flcfr/f30czndR8Ph88Hg+8Xq82547RwYMHER0dbT2GCbWrnY3a1c5G7XztrN2A2tXOR+1qZ8Pc3ltHu1ekD4QQc2VlZdYjmFE7J7VzUjsntfNh7QbUzkrtnNTOibndadqcE3MHDhywHsGM2jmpnZPaOamdD2s3oHZWauekdk7M7U7T5pyYY35ZrNo5qZ2T2jmpnQ9rN6B2VmrnpHZOzO1O03vOHSd6z7nea29vR3h4uPUYJtSudjZqVzsbtfO1s3YDalc7H7WrnQ1ze2/pPefkhFFUVGQ9ghm1c1I7J7VzUjsf1m5A7azUzkntnJjbnabNORERERERERERESPanBNzAwcOtB7BjNo5qZ2T2jmpnQ9rN6B2VmrnpHZOzO1O0+acmIuIiLAewYzaOamdk9o5qZ0PazegdlZq56R2TsztTtPmnJgrKyuzHsGM2jmpnZPaOamdD2s3oHZWauekdk7M7U7T5pyIiIiIiIiIiIgRl9/v91sPcTI42o/HlcM1NzcjJibGegwTalc7G7WrnY3a+dpZuwG1q52P2tXOhrm9t452r0ivnBNz5eXl1iOYUTsntXNSOye182HtBtTOSu2c1M6Jud1p2pwTcw0NDdYjmFE7J7VzUjsntfNh7QbUzkrtnNTOibndadqcE3ORkZHWI5hROye1c1I7J7XzYe0G1M5K7ZzUzom53Wl6z7njRO8513tdXV0ICeHcJ1a72tmoXe1s1M7XztoNqF3tfNSudjbM7b2l95yTE8aaNWusRzCjdk5q56R2Tmrnw9oNqJ2V2jmpnRNzu9O0OSciIiIiIiIiImJEm3NiLi0tzXoEM2rnpHZOauekdj6s3YDaWamdk9o5Mbc7TZtzYi42NtZ6BDNq56R2TmrnpHY+rN2A2lmpnZPaOTG3O02bc2Jux44d1iOYUTsntXNSOye182HtBtTOSu2c1M6Jud1p2pwTEREREREREREx4vL7/X7rIU4GR/vxuHK4AwcOoF+/ftZjmFC72tmoXe1s1M7XztoNqF3tfNSudjbM7b11tHtFYX04k8gRVVVV0f4fXO1qZ6N2tbNRO187azegdrUHlwJXgeP3kXJhCva9ss/R+1jsX+zo7fdWsK57X1A7Z7vT9GutYq6urs56BDNq56R2TmrnpHY+rN2A2lkxt8fm8b45PvO6q12coM05MRceHm49ghm1c1I7J7VzUjsf1m5A7ayY2zsbO61HMMO87moXJ+g9544TveeciIiIiIhIcOiLX2vtC8H6a60icnSOdq9Ir5wTc6tWrbIewYzaOamdk9o5qZ0PazegdlbM7Tm35ViPYIZ53dUuTtDmnIiIiIiIiMixclkPICInC23OibmUlBTrEcyonZPaOamdk9r5sHYDamfF3O5b67MewQzzuqtdnKDNOTHn8XisRzCjdk5q56R2Tmrnw9oNqJ0Vc/vBTw5aj2CGed3VLk7Q5pyY27Ztm/UIZtTOSe2c1M5J7XxYuwG1s2JuT70k1XoEM8zrrnZxgjbnREREREREREREjGhzTsyNGDHCegQzauekdk5q56R2PqzdgNpZMbdX/a7KegQzzOuudnGCNufEXG1trfUIZtTOSe2c1M5J7XxYuwG1s2JujxsdZz2CGeZ1V7s4QZtzYo75/+Bq56R2TmrnpHY+rN2A2lkxt8eN0eYcI7WLE7Q5J+ZCQngPQ7VzUjsntXNSOx/WbkDtrJjbu9q6rEcww7zuahcnuPx+v996iJOBz+eDx+OB1+uF2+22HkdERERERIRWgavAeoTjYrF/sfUIIvIfONq9ItNtz+XLl+P8889Heno6XC4XXn311cOuU1paigsuuAAejwexsbGYPHkydu/eHbi8paUF1157Lfr374+4uDhcdNFFqK6u7nEbu3fvxnnnnYeYmBikpKRg4cKF6Ojo6HGd999/HxMnTkRkZCSGDBmCZ5991olkOYI1a9ZYj2BG7ZzUzkntnNTOh7UbUDsr5vasm7OsRzDDvO5qFyeYbs41NTVh3LhxeOyxx454+Y4dO3DaaadhxIgReP/997F+/Xr87Gc/Q1RUVOA6P/zhD/HXv/4VL7/8Mj744ANUVlbi61//euDyzs5OnHfeeWhra8OKFSvw3HPP4dlnn8WiRYsC1ykrK8N5552Hs88+G8XFxViwYAG++93v4q233nIuXgK6unhfDq52TmrnpHZOaufD2g2onRVze0gE76/4Ma+72sUJYZZ3Pnv2bMyePfszL//JT36Cc889F0uWLAl8LTc3N/C/vV4vfvOb3+DFF1/El7/8ZQDAM888g7y8PHz00Uf40pe+hLfffhubNm3CP/7xD6SmpmL8+PG48847ccstt+D2229HREQEnnjiCeTk5OBXv/oVACAvLw///Oc/8cADD2DWrFkO1Uu3pKQk6xHMqJ2T2jmpnZPa+bB2A2pnxdzeuKHRegQzzOuudnFC0G71d3V14W9/+xuGDRuGWbNmISUlBVOnTu3xq69r165Fe3s7ZsyYEfjaiBEjMGjQIKxcuRIAsHLlSowZMwapqamB68yaNQs+nw8lJSWB63z6Nrqv030bR9La2gqfz9fjP+kd5v+Dq52T2jmpnZPa+bB2A2pnxdzeuFGbc4zULk4wfeXc59m3bx8aGxvxy1/+Ej//+c9xzz334M0338TXv/51vPfeezjzzDNRVVWFiIgIxMfH9/i7qampqKqqAgBUVVX12Jjrvrz7ss+7js/nw8GDBxEdHX3YfHfffTcKCg5/k9HCwkLExsZi4sSJKC0txcGDB9GvXz/k5ORg/fr1AICsrCx0dXWhvLwcADB+/Hhs374djY2NiI2NxbBhw7Bu3ToAQEZGBkJDQ7Fr1y4AwNixY7Fz5074fD5ERUVh1KhRWLt2LQAgPT0dUVFR+OSTTwAAo0ePRkVFBRoaGhAREYHx48dj9erVAIC0tDTExcVh+/btAA69WrC6uhp1dXUICwvDpEmTsHr1avj9fiQnJyMhIQFbt24FAAwfPhx1dXWoqalBSEgIJk+ejMLCQnR2dqJ///5ISUlBaWkpAGDo0KHw+XyB9wGcOnUqioqK0N7ejoSEBKSnp2PlypVISEhAbm4umpubsXfvXgBAfn4+Nm7ciJaWFng8HgwaNAgbNmwAAGRnZ6OjowMVFRUAgIkTJ2Lz5s1obm5GXFwccnNz8fHHHwMABg0aBACB9yocN24cduzYgcbGRsTExGDEiBEoKioKPN5hYWHYuXMnAGDMmDHYvXs3vF4voqKiMHr0aBQWFgIABgwYgJiYGOzYsQMAMGrUKFRWVqK+vh7h4eGYOHEiVq1aFTie3G43tm3bFni89+3bh+3btyMpKQn5+flYs2YNurq6kJycjMTERGzZsgUAMGzYMNTX16OmpgYulwtTpkzB2rVr0dHRgcTERKSmpgYe7yFDhqCxsTFwbE+ZMgXFxcVoa2tDfHw8MjIysHHjRgDA4MGD0dLSgsrKSgDApEmTUFJSgpaWFrjdbmRnZ/c4Zjs7OwOP94QJE7B161Y0NTUhLi4OQ4YMQXFxMQAgMzMTISEhPY7ZsrIyHDhwANHR0cjLy0NRURHq6+sxevRoREREoKysLPB4l5eXo6GhAZGRkRg7dmzgfQ3S0tIQGxsbeLxHjhyJqqoq1NXVHfZ4p6SkwOPxBB7vESNGoLa2FrW1tYFjtvvxTkpKQlJSEjZv3hw4Zr1eL/bt23fYMZuYmIi0tDRs2rQJwKFX8jY1NQUe78mTJ2P9+vVobW1FfHw8MjMzA8dsTk4O2trasGfPHtTX12P69OmU54impibk5+cf0zmi+x9STvRzREdHB5KTk4/pHLF//36Ehoae8OcIn88X+HSvoz1HAMDAgQNP+HPEhg0bEBMTA+DozxHdx+yJfo7ofn53+vuIYDtH7NixA5GRkYFj1snvI4LtHLF69Wq43e4++T4CCK5zRPfx3hffR3Qfs8FyjigrKwv8hlIw/ayRclEKGlY0YOBVAwEANa/VIDwpHPGnxAMAdt67EwO/MxDh/cNx8JOD2P+P/ciYnwEAqP17LUJjQpFwZgIAYNeDuzDg8gGISI1AS0ULav5Sg8xrMxEzNAYVTxw6rhOnJwIAyh8rR/LXkhGVEYW26jbs/d1eZC049N509R/Uo7O5E0mzD21wVCytQP8Z/RE9OBrt+9ux5+k9yF6YDQBoWNGA9tr2wHEZbOeI+vp6zJo1K2h/1gCcO0c0Nzdj3LhxQfuzBuDcOaKzsxMJCQlB+7NGMH4fkZBw6DzyRYLm01pdLhdeeeUVzJkzBwBQWVmJgQMH4vLLL8eLL74YuN4FF1yA2NhY/O53v8OLL76Iq666Cq2trT1ua8qUKTj77LNxzz33YP78+di1a1eP949rbm5GbGws3njjDcyePRvDhg3DVVddhdtuuy1wnTfeeAPnnXcempubj7g519ra2uN+fT4fMjMz9WmtvbBq1SpMnTrVegwTalc7G7WrnY3a+dpZuwG1qz249MWnteb8OAdld5U5eh/B+mmtwbrufUHtnO29dUJ8WuvnSUpKQlhYGEaOHNnj63l5eYF/uUxLS0NbWxsaGhp6XKe6uhppaWmB6/z7p7d2//mLruN2u4+4MQcAkZGRgX8Z7P5Pemfo0KHWI5hROye1c1I7J7XzYe0G1M6Kub365eovvtJJinnd1S5OCNrNuYiICEyePDnwcttuW7duRVbWoZcFT5o0CeHh4XjnnXcCl2/ZsgW7d+/GtGnTAADTpk3Dhg0bAi8bBYBly5bB7XYHNv6mTZvW4za6r9N9G+Isr9drPYIZtXNSOye1c1I7H9ZuQO2smNujBx/5hRwMmNdd7eIE0825xsZGFBcXB35/vKysDMXFxYFXxi1cuBAvvfQSnnrqKWzfvh2PPvoo/vrXv+L73/8+AMDj8WDevHm48cYb8d5772Ht2rW46qqrMG3aNHzpS18CAMycORMjR47EFVdcgY8//hhvvfUWfvrTn+Laa68NvB/I9773PXzyySf40Y9+hM2bN+Pxxx/HH/7wB/zwhz/s+weF0Kc3TtmonZPaOamdk9r5sHYDamfF3O6exPvbU8zrrnZxgukHQhQWFuLss88O/PnGG28EAFx55ZV49tlnceGFF+KJJ57A3XffjRtuuAHDhw/Hn/70J5x22mmBv/PAAw8gJCQEF110EVpbWzFr1iw8/vjjgctDQ0Px+uuv45prrsG0adMQGxuLK6+8EnfccUfgOjk5Ofjb3/6GH/7wh3jooYeQkZGB//7v/8asWbP64FEQERERERGRE05QvHu7iJwMguYDIU50R/smfyIiIiIiIuKsvvhAiL4QrB8IISJH54T/QAjh0f1R14zUzkntnNTOSe18WLsBtbNibh90wyDrEcwwr7vaxQnanBNz7e3t1iOYUTsntXNSOye182HtBtTOirk9NC7UegQzzOuudnGCNufEXGJiovUIZtTOSe2c1M5J7XxYuwG1s2Jubyptsh7BDPO6q12coM05MZeWlmY9ghm1c1I7J7VzUjsf1m5A7ayY272rvdYjmGFed7WLE7Q5J+Y2bdpkPYIZtXNSOye1c1I7H9ZuQO2smNvTr0y3HsEM87qrXZygzTkREREREREREREj2pwTc7m5udYjmFE7J7VzUjsntfNh7QbUzoq5fd+r+6xHMMO87moXJ2hzTsw1NfG+karaOamdk9o5qZ0PazegdlbM7ZFpkdYjmGFed7WLE7Q5J+aqqqqsRzCjdk5q56R2Tmrnw9oNqJ0Vc7vnSx7rEcwwr7vaxQnanBMRERERERERETHi8vv9fushTgY+nw8ejwderxdut9t6nBNKV1cXQkI494nVrnY2alc7G7XztbN2A2pXe3ApcBU4fyehADqdvYvF/sXO3kEvBeu69wW1c7b31tHuFelRFXPr16+3HsGM2jmpnZPaOamdD2s3oHZWzO0Z8zOsRzDDvO5qFydoc07Mtba2Wo9gRu2c1M5J7ZzUzoe1G1A7K+b28IRw6xHMMK+72sUJ2pwTc/Hx8dYjmFE7J7VzUjsntfNh7QbUzoq5vXl7s/UIZpjXXe3iBG3OibnMzEzrEcyonZPaOamdk9r5sHYDamfF3F7/fr31CGaY113t4gRtzom5DRs2WI9gRu2c1M5J7ZzUzoe1G1A7K+b2gd8daD2CGeZ1V7s4QZtzIiIiIiIiIiIiRrQ5J+ZycnKsRzCjdk5q56R2Tmrnw9oNqJ0Vc3vtG7XWI5hhXne1ixO0OSfm2trarEcwo3ZOauekdk5q58PaDaidFXN7aL9Q6xHMMK+72sUJ2pwTc3v27LEewYzaOamdk9o5qZ0PazegdlbM7QmnJ1iPYIZ53dUuTtDmnIiIiIiIiIiIiBGX3+/3Ww9xMvD5fPB4PPB6vXC73dbjnFDa29sRHh5uPYYJtaudjdrVzkbtfO2s3YDa1R5cClwFjt9HSFQIulq6HL2Pxf7Fjt5+bwXruvcFtXO299bR7hXplXNirrS01HoEM2rnpHZOauekdj6s3YDaWTG3D7higPUIZpjXXe3iBG3OibmDBw9aj2BG7ZzUzkntnNTOh7UbUDsr5vaI5AjrEcwwr7vaxQnanBNz/fr1sx7BjNo5qZ2T2jmpnQ9rN6B2VsztLbtbrEcww7zuahcnaHNOzOXk5FiPYEbtnNTOSe2c1M6HtRtQOyvm9pq/1ViPYIZ53dUuTtDmnJhbv3699Qhm1M5J7ZzUzkntfFi7AbWzYm7PvCbTegQzzOuudnGCNudERERERERERESMaHNOzGVlZVmPYEbtnNTOSe2c1M6HtRtQOyvm9v3L9luPYIZ53dUuTtDmnJjr6uqyHsGM2jmpnZPaOamdD2s3oHZWzO2uUJf1CGaY113t4gRtzom58vJy6xHMqJ2T2jmpnZPa+bB2A2pnxdye+OVE6xHMMK+72sUJ2pwTEREREREREREx4vL7/X7rIU4GPp8PHo8HXq8XbrfbepwTSmtrKyIjI63HMKF2tbNRu9rZqJ2vnbUbULvag0uBq8Dx+wjtF4rOA52O3sdi/2JHb7+3gnXd+4LaOdt762j3ivTKOTG3fft26xHMqJ2T2jmpnZPa+bB2A2pnxdye+vVU6xHMMK+72sUJ2pwTc42NjdYjmFE7J7VzUjsntfNh7QbUzoq5PXIg7yuImNdd7eIEbc6JudjYWOsRzKidk9o5qZ2T2vmwdgNqZ8Xc3rq31XoEM8zrrnZxgt5z7jjRe871XltbGyIiIqzHMKF2tbNRu9rZqJ2vnbUbULvag0ufvOdcXCg6Gznfcy5Y170vqJ2zvbf0nnNywli3bp31CGbUzkntnNTOSe18WLsBtbNibh90wyDrEcwwr7vaxQnanBMRERERERERETGizTkxl5GRYT2CGbVzUjsntXNSOx/WbkDtrJjb696vsx7BDPO6q12coM05MRcaGmo9ghm1c1I7J7VzUjsf1m5A7ayY2/2tvG/fzrzuahcnaHNOzO3atct6BDNq56R2TmrnpHY+rN2A2lkxt/ef1d96BDPM6652cYI250RERERERERERIy4/H4/72txj6Oj/XhcOdzBgwcRHR1tPYYJtaudjdrVzkbtfO2s3YDa1R5cClwFjt9HeP9wtO9vd/Q+FvsXO3r7vRWs694X1M7Z3ltHu1ekV86JuZ07d1qPYEbtnNTOSe2c1M6HtRtQOyvmduZfa2Ved7WLE7Q5J+Z8Pp/1CGbUzkntnNTOSe18WLsBtbNibo/O5n0FEfO6q12coM05MRcVFWU9ghm1c1I7J7VzUjsf1m5A7ayY29tq26xHMMO87moXJ+g9544Tvedc73V0dCAsLMx6DBNqVzsbtaudjdr52lm7AbWrPbj0xXvOuSJc8Lc5++N0sL7nXLCue19QO2d7b+k95+SEsXbtWusRzKidk9o5qZ2T2vmwdgNqZ8Xcnn1ztvUIZpjXXe3iBG3OiYiIiIiIiIiIGNHmnJhLT0+3HsGM2jmpnZPaOamdD2s3oHZWzO0N/2qwHsEM87qrXZxgujm3fPlynH/++UhPT4fL5cKrr776mdf93ve+B5fLhQcffLDH1+vq6jB37ly43W7Ex8dj3rx5aGxs7HGd9evX4/TTT0dUVBQyMzOxZMmSw27/5ZdfxogRIxAVFYUxY8bgjTfeOB6JchSY31RS7ZzUzkntnNTOh7UbUDsr5vb2+nbrEcwwr7vaxQmmm3NNTU0YN24cHnvssc+93iuvvIKPPvroiLu0c+fORUlJCZYtW4bXX38dy5cvx/z58wOX+3w+zJw5E1lZWVi7di3uvfde3H777Vi6dGngOitWrMDll1+OefPmYd26dZgzZw7mzJmDjRs3Hr9Y+UyffPKJ9Qhm1M5J7ZzUzkntfFi7AbWzYm5P/mqy9QhmmNdd7eIE04/ZmD17NmbPnv2519mzZw+uv/56vPXWWzjvvPN6XFZaWoo333wTa9asQX5+PgDgkUcewbnnnov77rsP6enpeOGFF9DW1oann34aERERGDVqFIqLi3H//fcHNvEeeughnHPOOVi4cCEA4M4778SyZcvw6KOP4oknnnCgXEREREREREREJMjfc66rqwtXXHEFFi5ciFGjRh12+cqVKxEfHx/YmAOAGTNmICQkBKtWrQpc54wzzkBERETgOrNmzcKWLVtQX18fuM6MGTN63PasWbOwcuXKz5yttbUVPp+vx3/SO6NHj7YewYzaOamdk9o5qZ0PazegdlbM7Xue3mM9ghnmdVe7OMH0lXNf5J577kFYWBhuuOGGI15eVVWFlJSUHl8LCwtDYmIiqqqqAtfJycnpcZ3U1NTAZQkJCaiqqgp87dPX6b6NI7n77rtRUFBw2NcLCwsRGxuLiRMnorS0FAcPHkS/fv2Qk5OD9evXAwCysrLQ1dWF8vJyAMD48eOxfft2NDY2IjY2FsOGDcO6desAABkZGQgNDcWuXbsAAGPHjsXOnTvh8/kQFRWFUaNGBT7OOD09HVFRUYGXmo4ePRoVFRVoaGhAREQExo8fj9WrVwMA0tLSEBcXh+3btwMA8vLyUF1djbq6OoSFhWHSpElYvXo1/H4/kpOTkZCQgK1btwIAhg8fjrq6OtTU1CAkJASTJ09GYWEhOjs70b9/f6SkpKC0tBQAMHToUPh8PlRXVwMApk6diqKiIrS3tyMhIQHp6elYtWoV4uLikJubi+bmZuzduxcAkJ+fj40bN6KlpQUejweDBg3Chg0bAADZ2dno6OhARUUFAGDixInYvHkzmpubA7f18ccfAwAGDRoEANi9ezcAYNy4cdixYwcaGxsRExODESNGoKioKPB4h4WFYefOnQCAMWPGYPfu3fB6vYiKisLo0aNRWFgIABgwYABiYmKwY8cOAMCoUaNQWVmJ+vp6hIeHY+LEiYFN4tTUVLjdbmzbti3weO/btw+7du2Cx+NBfn4+1qxZg66uLiQnJyMxMRFbtmwBAAwbNgz19fWoqamBy+XClClTsHbtWnR0dCAxMRGpqamBx3vIkCFobGwMHLtTpkxBcXEx2traEB8fj4yMjMCvaw8ePBgtLS2orKwEAEyaNAklJSVoaWmB2+1GdnZ2j2O2s7Mz8HhPmDABW7duRVNTE+Li4jBkyBAUFxcDADIzMxESEtLjmC0rK8OBAwcQHR2NvLw8FBUVobGxEcOHD0dERATKysoCj3d5eTkaGhoQGRmJsWPHYs2aNYFjNjY2NvB4jxw5ElVVVairqzvs8U5JSYHH4wk83iNGjEBtbS1qa2sDx2z3452UlISkpCRs3rw5cMx6vV7s27fvsGM2MTERaWlp2LRpEwAgNzcXTU1Ngcd78uTJWL9+PVpbWxEfH4/MzMzAMZuTk4O2tjbs2bMHjY2NOOOMMyjPEZ2dnRg9evQxnSNKSkoCj/eJfI6IiYlBZGTkMZ0j9u/fj9DQ0BP+HAEg8H6wR3uOAICBAwee8OeI7nmAoz9HdB+zJ/o54qOPPkJcXJzj30cE2zmiqqoKHR0dgWPWye8jgu0cUVhYiKioqD75PgIIrnNEYWEh4uLi+uT7iO5jNljOERUVFUhMTAy6nzVSLkpBw4oGDLxqIACg5rUahCeFI/6UeADAznt3YuB3BiK8fzgOfnIQ+/+xHxnzMwAAtX+vRWhMKBLOTAAA7HpwFwZcPgARqRFoqWhBzV9qkHltJiLTI7H3t4fOOYnTEwEA5Y+VI/lryYjKiEJbdRv2/m4vshZkAQDqP6hHZ3MnkmYnAQAqllag/4z+iB4cjfb97djz9B5kL8wGADSsaEB7bXvguAy2c0RjYyO+/OUvB+3PGoBz5wi/349hw4YF7c8agHPniNjYWISGhgbtzxrB+H1EQsKh88gXcfn9fv9RXdNhLpcLr7zyCubMmQMAWLt2Lc477zwUFRUF3msuOzsbCxYswIIFCwAAd911F5577rnAiaVbSkoKCgoKcM0112DmzJnIycnBk08+Gbh806ZNGDVqFDZt2oS8vDxERETgueeew+WXXx64zuOPP46CgoLAIv671tZWtLa2Bv7s8/mQmZkJr9cLt9t9PB4SGqtWrcLUqVOtxzChdrWzUbva2aidr521G1C72oNLgevwF1Icbzk/zkHZXWWO3sdi/2JHb7+3gnXd+4LaOdt7y+fzwePxfOFeUdD+WuuHH36Iffv2YdCgQQgLC0NYWBh27dqFm266CdnZ2QAO7bZ27zh36+joQF1dHdLS0gLX+fcNtu4/f9F1ui8/ksjISLjd7h7/Se98+leO2aidk9o5qZ2T2vmwdgNqZ8Xc3uHtsB7BDPO6q12cELSbc1dccQXWr1+P4uLiwH/p6elYuHAh3nrrLQDAtGnT0NDQEHgZJQC8++676OrqCuzmTps2DcuXL0d7+/99zPWyZcswfPjwwMsLp02bhnfeeafH/S9btgzTpk1zOlNw6GW0rNTOSe2c1M5J7XxYuwG1s2JuL/91ufUIZpjXXe3iBNPNucbGxsDGGwCUlZWhuLgYu3fvRv/+/TF69Oge/4WHhyMtLQ3Dhw8HcOj3ks855xxcffXVWL16Nf71r3/huuuuw2WXXRb4VdhvfOMbiIiIwLx581BSUoKXXnoJDz30EG688cbAHD/4wQ/w5ptv4le/+hU2b96M22+/HYWFhbjuuuv6/DFh1P1754zUzkntnNTOSe18WLsBtbNibs+5NeeLr3SSYl53tYsTTDfnCgsLMWHCBEyYMAEAcOONN2LChAlYtGjRUd/GCy+8gBEjRmD69Ok499xzcdppp2Hp0qWByz0eD95++22UlZVh0qRJuOmmm7Bo0SLMnz8/cJ1TTjkFL774IpYuXYpx48bhj3/8I1599VV9EomIiIiIiIiIiDjK9NNazzrrLBzL51F0fzrepyUmJuLFF1/83L83duxYfPjhh597nUsuuQSXXHLJUc8ix8/nvbffyU7tnNTOSe2c1M6HtRtQOyvmdu9qr/UIZpjXXe3ihKB9zznhERcXZz2CGbVzUjsntXNSOx/WbkDtrJjbW/e0Wo9ghnnd1S5O0OacmNu+fbv1CGbUzkntnNTOSe18WLsBtbNibk+5MMV6BDPM6652cYI250RERERERERERIxoc07M5eXlWY9gRu2c1M5J7ZzUzoe1G1A7K+b2yucrrUcww7zuahcnaHNOzFVXV1uPYEbtnNTOSe2c1M6HtRtQOyvmdne+23oEM8zrrnZxgjbnxFxdXZ31CGbUzkntnNTOSe18WLsBtbNibo8byfvm+MzrrnZxgjbnxFxYWJj1CGbUzkntnNTOSe18WLsBtbNibu9s7rQewQzzuqtdnODy+/1+6yFOBj6fDx6PB16vF24378ubRURERERErBW4CqxHOC4W+xdbjyAi/4Gj3SvSK+fE3OrVq61HMKN2TmrnpHZOaufD2g2onRVze/Yt2dYjmGFed7WLE7Q5J+aYX7ypdk5q56R2Tmrnw9oNqJ0Vc7sr1GU9ghnmdVe7OEGbc2IuOTnZegQzauekdk5q56R2PqzdgNpZMbcfWHfAegQzzOuudnGCNufEXEJCgvUIZtTOSe2c1M5J7XxYuwG1s2Jub9rWZD2CGeZ1V7s4QZtzYm7r1q3WI5hROye1c1I7J7XzYe0G1M6KuT3t0jTrEcwwr7vaxQnanBMRERERERERETGizTkxN3z4cOsRzKidk9o5qZ2T2vmwdgNqZ8XcXvX7KusRzDCvu9rFCdqcE3N1dXXWI5hROye1c1I7J7XzYe0G1M6KuT02L9Z6BDPM6652cYI258RcTU2N9Qhm1M5J7ZzUzkntfFi7AbWzYm7vN66f9QhmmNdd7eIEbc6JuZAQ3sNQ7ZzUzkntnNTOh7UbUDsr5nZ/u996BDPM6652cYLL7/fznlGOI5/PB4/HA6/XC7fbbT2OiIiIiIgIrQJXgfUIx8Vi/2LrEUTkP3C0e0Xa9hRzhYWF1iOYUTsntXNSOye182HtBtTOirk968Ys6xHMMK+72sUJ2pwTc52dndYjmFE7J7VzUjsntfNh7QbUzoq5PSSK98dp5nVXuziB92wiQaN///7WI5hROye1c1I7J7XzYe0G1M6Kub2xpNF6BDPM6652cYI258RcSkqK9Qhm1M5J7ZzUzkntfFi7AbWzYm4/UHzAegQzzOuudnGCNufEXGlpqfUIZtTOSe2c1M5J7XxYuwG1s2JuHzB3gPUIZpjXXe3iBG3OiYiIiIiIiIiIGNHmnJgbOnSo9Qhm1M5J7ZzUzkntfFi7AbWzYm6v/lO19QhmmNdd7eIEbc6JOZ/PZz2CGbVzUjsntXNSOx/WbkDtrJjbo7OirUcww7zuahcnaHNOzFVX8/6Lk9o5qZ2T2jmpnQ9rN6B2Vszt7ny39QhmmNdd7eIEbc6JiIiIiIiIiIgYcfn9fr/1ECcDn88Hj8cDr9cLt5v3X1BERERERESsFbgKrEc4Lhb7F1uPICL/gaPdK9Ir58RcUVGR9Qhm1M5J7ZzUzkntfFi7AbWzYm7PvD7TegQzzOuudnGCNufEXHt7u/UIZtTOSe2c1M5J7XxYuwG1s2JuD+sXZj2CGeZ1V7s4QZtzYi4hIcF6BDNq56R2TmrnpHY+rN2A2lkxtzdtabIewQzzuqtdnKDNOTGXnp5uPYIZtXNSOye1c1I7H9ZuQO2smNsbVjRYj2CGed3VLk7Q5pyYKykpsR7BjNo5qZ2T2jmpnQ9rN6B2VsztA68aaD2CGeZ1V7s4QZtzIiIiIiIiIiIiRrQ5J+Zyc3OtRzCjdk5q56R2Tmrnw9oNqJ0Vc3vNazXWI5hhXne1ixO0OSfmmpubrUcwo3ZOauekdk5q58PaDaidFXN7eFK49QhmmNdd7eIEbc6Jub1791qPYEbtnNTOSe2c1M6HtRtQOyvm9vhT4q1HMMO87moXJ2hzTkRERERERERExIjL7/f7rYc4Gfh8Png8Hni9XrjdbutxTiidnZ0IDQ21HsOE2tXORu1qZ6N2vnbWbkDtag8uBa4Cx+/DFe6Cv93ZH6cX+xc7evu9Fazr3hfUztneW0e7V6RXzom5jRs3Wo9gRu2c1M5J7ZzUzoe1G1A7K+b2gd8ZaD2CGeZ1V7s4QZtzYq6lpcV6BDNq56R2TmrnpHY+rN2A2lkxt4f35/1ACOZ1V7s4QZtzYs7j8ViPYEbtnNTOSe2c1M6HtRtQOyvm9oOfHLQewQzzuqtdnKDNOTE3aNAg6xHMqJ2T2jmpnZPa+bB2A2pnxdy+/x/7rUcww7zuahcnaHNOzG3YsMF6BDNq56R2TmrnpHY+rN2A2lkxt2fMz7AewQzzuqtdnKDNORERERERERERESPanBNz2dnZ1iOYUTsntXNSOye182HtBtTOirm99u+11iOYYV53tYsTtDkn5jo6OqxHMKN2TmrnpHZOaufD2g2onRVze2hMqPUIZpjXXe3iBG3OibmKigrrEcyonZPaOamdk9r5sHYDamfF3J5wZoL1CGaY113t4gRtzomIiIiIiIiIiBgx3Zxbvnw5zj//fKSnp8PlcuHVV18NXNbe3o5bbrkFY8aMQWxsLNLT0/Gtb30LlZWVPW6jrq4Oc+fOhdvtRnx8PObNm4fGxsYe11m/fj1OP/10REVFITMzE0uWLDlslpdffhkjRoxAVFQUxowZgzfeeMORZjncxIkTrUcwo3ZOauekdk5q58PaDaidFXP7rgd3WY9ghnnd1S5OMN2ca2pqwrhx4/DYY48ddllzczOKiorws5/9DEVFRfjzn/+MLVu24IILLuhxvblz56KkpATLli3D66+/juXLl2P+/PmBy30+H2bOnImsrCysXbsW9957L26//XYsXbo0cJ0VK1bg8ssvx7x587Bu3TrMmTMHc+bMwcaNG52Ll4DNmzdbj2BG7ZzUzkntnNTOh7UbUDsr5vYBlw+wHsEM87qrXZwQZnnns2fPxuzZs494mcfjwbJly3p87dFHH8WUKVOwe/duDBo0CKWlpXjzzTexZs0a5OfnAwAeeeQRnHvuubjvvvuQnp6OF154AW1tbXj66acRERGBUaNGobi4GPfff39gE++hhx7COeecg4ULFwIA7rzzTixbtgyPPvoonnjiCQcfAQEObcSyUjsntXNSOye182HtBtTOirk9IjXCegQzzOuudnHCCfWec16vFy6XC/Hx8QCAlStXIj4+PrAxBwAzZsxASEgIVq1aFbjOGWecgYiI/ztxzpo1C1u2bEF9fX3gOjNmzOhxX7NmzcLKlSs/c5bW1lb4fL4e/0nvxMXFWY9gRu2c1M5J7ZzUzoe1G1A7K+b2looW6xHMMK+72sUJpq+cOxYtLS245ZZbcPnll8PtdgMAqqqqkJKS0uN6YWFhSExMRFVVVeA6OTk5Pa6TmpoauCwhIQFVVVWBr336Ot23cSR33303CgoKDvt6YWEhYmNjMXHiRJSWluLgwYPo168fcnJysH79egBAVlYWurq6UF5eDgAYP348tm/fjsbGRsTGxmLYsGFYt24dACAjIwOhoaHYtevQ+xmMHTsWO3fuhM/nQ1RUFEaNGoW1a9cCANLT0xEVFYVPPvkEADB69GhUVFSgoaEBERERGD9+PFavXg0ASEtLQ1xcHLZv3w4AyMvLQ3V1Nerq6hAWFoZJkyZh9erV8Pv9SE5ORkJCArZu3QoAGD58OOrq6lBTU4OQkBBMnjwZhYWF6OzsRP/+/ZGSkoLS0lIAwNChQ+Hz+VBdXQ0AmDp1KoqKitDe3o6EhASkp6fD6/Vi1apVyM3NRXNzM/bu3QsAyM/Px8aNG9HS0gKPx4NBgwZhw4YNAIDs7Gx0dHQEPi1m4sSJ2Lx5M5qbmxEXF4fc3Fx8/PHHAIBBgwYBAHbv3g0AGDduHHbs2IHGxkbExMRgxIgRKCoqCjzeYWFh2LlzJwBgzJgx2L17N7xeL6KiojB69GgUFhYCAAYMGICYmBjs2LEDADBq1ChUVlaivr4e4eHhmDhxYmCTODU1FW63G9u2bQs83vv27YPX60VhYSHy8/OxZs0adHV1ITk5GYmJidiyZQsAYNiwYaivr0dNTQ1cLhemTJmCtWvXoqOjA4mJiUhNTQ083kOGDEFjY2Pg2J0yZQqKi4vR1taG+Ph4ZGRkBH5de/DgwWhpaQm8j+OkSZNQUlKClpYWuN1uZGdn9zhmOzs7A4/3hAkTsHXrVjQ1NSEuLg5DhgxBcXExACAzMxMhISE9jtmysjIcOHAA0dHRyMvLQ1FRUeD2IiIiUFZWFni8y8vL0dDQgMjISIwdOxZr1qwJHLOxsbGBx3vkyJGoqqpCXV3dYY93SkoKPB5P4PEeMWIEamtrUVtbGzhmux/vpKQkJCUlBV6iPXToUHi9Xuzbt++wYzYxMRFpaWnYtGkTACA3NxdNTU2Bx3vy5MlYv349WltbER8fj8zMzMAxm5OTg7a2NuzZswednZ1ob2+nPEd4PB74fL5jOkeUlJQEHu8T+RwxZMgQbN269ZjOEfv370doaOgJf47IzMwMtB7tOQIABg4ceMKfI6KiogL3e7TniO5j9kQ/R/h8Pqxatcrx7yOC7RzhdrsDa+709xHBdo5obW3FqlWr+uT7CCC4zhHd38/2xfcR3cdssJwjvF4v1q1bF3Q/a6RclIKGFQ0YeNVAAEDNazUITwpH/CnxAICd9+7EwO8MRHj/cBz85CD2/2M/MuZnAABq/16L0JjQwKex7npwFwZcPgARqRFoqWhBzV9qkHltJlzhLnimegAAidMTAQDlj5Uj+WvJiMqIQlt1G/b+bi+yFmQBAOo/qEdncyeSZicBACqWVqD/jP6IHhyN9v3t2PP0HmQvzAYANKxoQHtte+C4DLZzRGdnJ/x+f9D+rAE4d45ITExEXV1d0P6sATh3jhg+fDhKS0uD9meNYPw+IiHh6D7V2eX3+/1HdU2HuVwuvPLKK5gzZ85hl7W3t+Oiiy5CRUUF3n///cDm3F133YXnnnsucGLplpKSgoKCAlxzzTWYOXMmcnJy8OSTTwYu37RpE0aNGoVNmzYhLy8PEREReO6553D55ZcHrvP444+joKAgsIj/rrW1Fa2trYE/+3w+ZGZmwuv1BuaTo7Nq1SpMnTrVegwTalc7G7WrnY3a+dpZuwG1qz24FLgOfyHF8Zbz4xyU3VXm6H0s9i929PZ7K1jXvS+onbO9t3w+HzwezxfuFQX9K+fa29tx6aWXYteuXXj33Xd7xKSlpQV2nLt1dHSgrq4OaWlpgev8+wZb95+/6Drdlx9JZGQkIiMjex8mIiIiIiIiIiL0gvo957o35rZt24Z//OMf6N+/f4/Lp02bhoaGhsDLKAHg3XffRVdXV2A3d9q0aVi+fDna29sD11m2bBmGDx8eeHnhtGnT8M477/S47WXLlmHatGlOpcmndP+6CCO1c1I7J7VzUjsf1m5A7ayY2+veqbMewQzzuqtdnGC6OdfY2Iji4uLA74+XlZWhuLgYu3fvRnt7Oy6++GIUFhbihRdeQGdnJ6qqqlBVVYW2tjYAh34v+ZxzzsHVV1+N1atX41//+heuu+46XHbZZUhPTwcAfOMb30BERATmzZuHkpISvPTSS3jooYdw4403Bub4wQ9+gDfffBO/+tWvsHnzZtx+++0oLCzEdddd1+ePiYiIiIiIiIiI8DDdnCssLMSECRMwYcIEAMCNN96ICRMmYNGiRdizZw9ee+01VFRUYPz48RgwYEDgvxUrVgRu44UXXsCIESMwffp0nHvuuTjttNOwdOnSwOUejwdvv/02ysrKMGnSJNx0001YtGgR5s+fH7jOKaecghdffBFLly7FuHHj8Mc//hGvvvoqRo8e3XcPBrHuN1hmpHZOauekdk5q58PaDaidFXN794dAMGJed7WLE0zfc+6ss87C530exdF8VkViYiJefPHFz73O2LFj8eGHH37udS655BJccsklX3h/IiIiIiIiIiIix0vQfFrrie5oP4FDDtfS0oKoqCjrMUyoXe1s1K52Nmrna2ftBtSu9uDSF5/WGuYJQ4e3w9H7CNZPaw3Wde8Lauds762j3SsK6g+EEA47duywHsGM2jmpnZPaOamdD2s3oHZWzO3JX0u2HsEM87qrXZygzTkx19jYaD2CGbVzUjsntXNSOx/WbkDtrJjbozJ4X0HEvO5qFydoc07MxcTEWI9gRu2c1M5J7ZzUzoe1G1A7K+b2tuo26xHMMK+72sUJes+540TvOdd77e3tCA8Ptx7DhNrVzkbtamejdr521m5A7WoPLn3xnnMhMSHoau5y9D6C9T3ngnXd+4LaOdt7S+85JyeMoqIi6xHMqJ2T2jmpnZPa+bB2A2pnxdyetSDLegQzzOuudnGCNudERERERERERESMaHNOzGVkZFiPYEbtnNTOSe2c1M6HtRtQOyvm9voP6q1HMMO87moXJ2hzTsyFhYVZj2BG7ZzUzkntnNTOh7UbUDsr5vbO5k7rEcwwr7vaxQnanBNzO3futB7BjNo5qZ2T2jmpnQ9rN6B2VsztSbOTrEcww7zuahcnaHNORERERERERETEiMvv9/uthzgZHO3H48rhmpubERMTYz2GCbWrnY3a1c5G7XztrN2A2tUeXApcBY7fR3hSONpr2x29j8X+xY7efm8F67r3BbVztvfW0e4V6ZVzYm737t3WI5hROye1c1I7J7XzYe0G1M6Kub3/jP7WI5hhXne1ixO0OSfmvF6v9Qhm1M5J7ZzUzkntfFi7AbWzYm6PHhxtPYIZ5nVXuzhBm3NiLioqynoEM2rnpHZOauekdj6s3YDaWTG3t+939ldagxnzuqtdnKD3nDtO9J5zvdfZ2YnQ0FDrMUyoXe1s1K52Nmrna2ftBtSu9uDSF+855wp3wd/u7I/Twfqec8G67n1B7ZztvaX3nJMTRmFhofUIZtTOSe2c1M5J7XxYuwG1s2Juz16YbT2CGeZ1V7s4QZtzIiIiIiIiIiIiRrQ5J+YGDBhgPYIZtXNSOye1c1I7H9ZuQO2smNsbVjRYj2CGed3VLk7Q5pyYi4mJsR7BjNo5qZ2T2jmpnQ9rN6B2Vszt7bW8HwjBvO5qFydoc07M7dixw3oEM2rnpHZOauekdj6s3YDaWTG3J1+QbD2CGeZ1V7s4QZtzIiIiIiIiIiIiRlx+v9/Zz34mcbQfjyuHa2xsRFxcnPUYJtSudjZqVzsbtfO1s3YDald7cClwFTh+HxEDItC2t83R+1jsX+zo7fdWsK57X1A7Z3tvHe1ekV45J+YqKyutRzCjdk5q56R2Tmrnw9oNqJ0Vc3v8KfHWI5hhXne1ixO0OSfm6uvrrUcwo3ZOauekdk5q58PaDaidFXN77PBY6xHMMK+72sUJ2pwTc+Hh4dYjmFE7J7VzUjsntfNh7QbUzoq5veNAh/UIZpjXXe3iBL3n3HGi95wTEREREREJDn3xnnN9IVjfc05Ejo7ec05OGKtWrbIewYzaOamdk9o5qZ0PazegdlbM7Tk/zrEewQzzuqtdnKDNORERERERERERESPanBNzqamp1iOYUTsntXNSOye182HtBtTOirndV+izHsEM87qrXZygzTkxx/wefWrnpHZOauekdj6s3YDaWTG3H9x10HoEM8zrrnZxgjbnxNy2bdusRzCjdk5q56R2Tmrnw9oNqJ0Vc3vqRbyvImJed7WLE7Q5JyIiIiIiIiIiYkSbc2IuLy/PegQzauekdk5q56R2PqzdgNpZMbfvfWGv9QhmmNdd7eIEbc6JuX379lmPYEbtnNTOSe2c1M6HtRtQOyvm9n7j+1mPYIZ53dUuTtDmnJjbv3+/9Qhm1M5J7ZzUzkntfFi7AbWzYm6PGxVnPYIZ5nVXuzhBm3NiLjQ01HoEM2rnpHZOauekdj6s3YDaWTG3d7V0WY9ghnnd1S5OcPn9fr/1ECcDn88Hj8cDr9erjxcWERERERExVOAqsB7huFjsX2w9goj8B452r0ivnBNza9assR7BjNo5qZ2T2jmpnQ9rN6B2Vszt2QuzrUcww7zuahcnaHNOzHV18b4cXO2c1M5J7ZzUzoe1G1A7K+Z2V7jLegQzzOuudnGCNufEXHJysvUIZtTOSe2c1M5J7XxYuwG1s2JuP/DxAesRzDCvu9rFCdqcE3OJiYnWI5hROye1c1I7J7XzYe0G1M6Kub2ptMl6BDPM6652cYI258Tcli1brEcwo3ZOauekdk5q58PaDaidFXN72mVp1iOYYV53tYsTtDknIiIiIiIiIiJiRJtzYm7YsGHWI5hROye1c1I7J7XzYe0G1M6Kub3qD1XWI5hhXne1ixO0OSfm6uvrrUcwo3ZOauekdk5q58PaDaidFXN77NBY6xHMMK+72sUJ2pwTczU1NdYjmFE7J7VzUjsntfNh7QbUzoq5vd+EftYjmGFed7WLE7Q5J+ZcLpf1CGbUzkntnNTOSe18WLsBtbNibvd3+q1HMMO87moXJ7j8fj/vGeU48vl88Hg88Hq9cLvd1uOIiIiIiIjQKnAVWI9wXCz2L7YeQUT+A0e7V6RXzom5tWvXWo9gRu2c1M5J7ZzUzoe1G1A7K+b2QQsGWY9ghnnd1S5O0OacmOvo6LAewYzaOamdk9o5qZ0PazegdlbM7aExodYjmGFed7WLE0w355YvX47zzz8f6enpcLlcePXVV3tc7vf7sWjRIgwYMADR0dGYMWMGtm3b1uM6dXV1mDt3LtxuN+Lj4zFv3jw0Njb2uM769etx+umnIyoqCpmZmViyZMlhs7z88ssYMWIEoqKiMGbMGLzxxhvHvVeOLDEx0XoEM2rnpHZOauekdj6s3YDaWTG3N25q/OIrnaSY113t4gTTzbmmpiaMGzcOjz322BEvX7JkCR5++GE88cQTWLVqFWJjYzFr1iy0tLQErjN37lyUlJRg2bJleP3117F8+XLMnz8/cLnP58PMmTORlZWFtWvX4t5778Xtt9+OpUuXBq6zYsUKXH755Zg3bx7WrVuHOXPmYM6cOdi4caNz8RKQmppqPYIZtXNSOye1c1I7H9ZuQO2smNt9hT7rEcwwr7vaxQmmm3OzZ8/Gz3/+c1x44YWHXeb3+/Hggw/ipz/9Kb72ta9h7NixeP7551FZWRl4hV1paSnefPNN/Pd//zemTp2K0047DY888gh+//vfo7KyEgDwwgsvoK2tDU8//TRGjRqFyy67DDfccAPuv//+wH099NBDOOecc7Bw4ULk5eXhzjvvxMSJE/Hoo49+5uytra3w+Xw9/pPeKS0ttR7BjNo5qZ2T2jmpnQ9rN6B2Vszt6d9Ktx7BDPO6q12cEGY9wGcpKytDVVUVZsyYEfiax+PB1KlTsXLlSlx22WVYuXIl4uPjkZ+fH7jOjBkzEBISglWrVuHCCy/EypUrccYZZyAiIiJwnVmzZuGee+5BfX09EhISsHLlStx444097n/WrFmH/Zrtp919990oKDj8E4AKCwsRGxuLiRMnorS0FAcPHkS/fv2Qk5OD9evXAwCysrLQ1dWF8vJyAMD48eOxfft2NDY2IjY2FsOGDcO6desAABkZGQgNDcWuXbsAAGPHjsXOnTvh8/kQFRWFUaNGBd6UMT09HVFRUfjkk08AAKNHj0ZFRQUaGhoQERGB8ePHY/Xq1QCAtLQ0xMXFYfv27QCAvLw8VFdXo66uDmFhYZg0aRJWr14Nv9+P5ORkJCQkYOvWrQCA4cOHo66uDjU1NQgJCcHkyZNRWFiIzs5O9O/fHykpKYH/0w4dOhQ+nw/V1dUAgKlTp6KoqAjt7e1ISEhAeno66uvrsWrVKuTm5qK5uRl79+4FAOTn52Pjxo1oaWmBx+PBoEGDsGHDBgBAdnY2Ojo6UFFRAQCYOHEiNm/ejObmZsTFxSE3Nxcff/wxAGDQoENv1Lp7924AwLhx47Bjxw40NjYiJiYGI0aMQFFRUeDxDgsLw86dOwEAY8aMwe7du+H1ehEVFYXRo0ejsLAQADBgwADExMRgx44dAIBRo0ahsrIS9fX1CA8Px8SJE7Fq1SoAh/6Fwe12B34tOy8vD/v27UN9fT0KCwuRn5+PNWvWoKurC8nJyUhMTMSWLVsAAMOGDUN9fT1qamrgcrkwZcoUrF27Fh0dHUhMTERqamrg8R4yZAgaGxtRVVUFAJgyZQqKi4vR1taG+Ph4ZGRkBF4ROnjwYLS0tAQ2sidNmoSSkhK0tLTA7XYjOzu7xzHb2dkZeLwnTJiArVu3oqmpCXFxcRgyZAiKi4sBAJmZmQgJCelxzJaVleHAgQOIjo5GXl4eioqKUF9fj4qKCkRERKCsrCzweJeXl6OhoQGRkZEYO3Ys1qxZEzhmY2NjA4/3yJEjUVVVhbq6usMe75SUFHg8nsDjPWLECNTW1qK2tjZwzHY/3klJSUhKSsLmzZsDx6zX68W+ffsOO2YTExORlpaGTZs2AQByc3PR1NQUeLwnT56M9evXo7W1FfHx8cjMzAwcszk5OWhra8OePXtQX1+P9vZ2ynNEU1MTfD7fMZ0jSkpKAo/3iXyO6OjowNatW4/pHLF//36Ehoae8OeIrq6uQOvRniMAYODAgSf8OaK5uTlwv0d7jug+Zk/0c0T387vT30cE2zmitbU1sOZOfx8RbOcIn8+HVatW9cn3EUBwnSO6j/e++D6i+5gNlnNEfX091q1bF3Q/a6RclIKGFQ0YeNVAAEDNazUITwpH/CnxAICd9+7EwO8MRHj/cBz85CD2/2M/MuZnAABq/16L0JhQJJyZAADY9eAuDLh8ACJSI9BS0YKav9Qg89pMxAyNgWeqBwCQOP3Qr/uVP1aO5K8lIyojCm3Vbdj7u73IWpAFAKj/oB6dzZ1Imp0EAKhYWoH+M/ojenA02ve3Y8/Te5C9MBsA0LCiAe217YHjMtjOEfX19fD7/UH7swbg3DmiubkZdXV1QfuzBuDcOaKzsxOlpaVB+7NGMH4fkZBw6DzyRVx+v99/VNd0mMvlwiuvvII5c+YAOPSrpqeeeioqKysxYMCAwPUuvfRSuFwuvPTSS7jrrrvw3HPPBU4s3VJSUlBQUIBrrrkGM2fORE5ODp588snA5Zs2bcKoUaOwadMm5OXlISIiAs899xwuv/zywHUef/xxFBQUBBbx37W2tqK1tTXwZ5/Ph8zMzC/8eFw53P79+9G/f3/rMUyoXe1s1K52Nmrna2ftBtSu9uBS4Dr8hRTHW2xeLJpKmxy9j8X+xY7efm8F67r3BbVztveWz+eDx+P5wr0ifVprL0VGRsLtdvf4T3rn3z/Ag4naOamdk9o5qZ0PazegdlbM7ZEDI61HMMO87moXJwTt5lxaWhoAHPbKterq6sBlaWlpgZeDduvo6EBdXV2P6xzpNj59H591ne7LxVndL9NlpHZOauekdk5q58PaDaidFXO7Z4rHegQzzOuudnFC0G7O5eTkIC0tDe+8807ga93vYzFt2jQAwLRp09DQ0BD4HWcAePfdd9HV1YWpU6cGrrN8+XK0t7cHrrNs2TIMHz488Lu/06ZN63E/3dfpvh8REREREREREREnmL7nXGNjY+ANACdMmID7778fZ599NhITEzFo0CDcc889+OUvf4nnnnsOOTk5+NnPfob169dj06ZNiIqKAnDoE1+rq6vxxBNPoL29HVdddRXy8/Px4osvAgC8Xi+GDx+OmTNn4pZbbsHGjRvxne98Bw888ADmz58P4ND725155pn45S9/ifPOOw+///3vcdddd6GoqAijR48+qpaj/T1iOZzf74fL5bIew4Ta1c5G7Wpno3a+dtZuQO1qDy598Z5zCAHQ5exdBOt7zgXruvcFtXO299YJ8Z5zhYWFmDBhAiZMmAAAuPHGGzFhwgQsWrQIAPCjH/0I119/PebPn4/JkyejsbERb775ZmBjDgBeeOEFjBgxAtOnT8e5556L0047DUuXLg1c7vF48Pbbb6OsrAyTJk3CTTfdhEWLFgU25gDglFNOwYsvvoilS5di3Lhx+OMf/4hXX331qDfm5D/T/ck7jNTOSe2c1M5J7XxYuwG1s2Juz7wm03oEM8zrrnZxQpjlnZ911ln4vBfuuVwu3HHHHbjjjjs+8zqJiYmBV8l9lrFjx+LDDz/83OtccskluOSSSz5/YHFEW1ub9Qhm1M5J7ZzUzkntfFi7AbWzYm4P85j+OG2Ked3VLk4I2vecEx7x8fHWI5hROye1c1I7J7XzYe0G1M6Kub15W7P1CGaY113t4gRtzom5jIwM6xHMqJ2T2jmpnZPa+bB2A2pnxdxe/2G99QhmmNdd7eIEbc6JuY0bN1qPYEbtnNTOSe2c1M6HtRtQOyvm9oHfGWg9ghnmdVe7OEGbcyIiIiIiIiIiIka0OSfmBg8ebD2CGbVzUjsntXNSOx/WbkDtrJjba16vsR7BDPO6q12coM05MdfS0mI9ghm1c1I7J7VzUjsf1m5A7ayY28MTwq1HMMO87moXJ2hzTsxVVlZaj2BG7ZzUzkntnNTOh7UbUDsr5vb4U+OtRzDDvO5qFydoc05ERERERERERMSIy+/3+62HOBn4fD54PB54vV643W7rcU4oHR0dCAsLsx7DhNrVziZY2wtcBY7fhyvCBX+bs0+5i/2LHb393grWde8LaudrZ+0G1K724KLndmcF67r3BbVztvfW0e4V6ZVzYq6kpMR6BDNq56R2TunfTrcewQzzuqudD2s3oHZWzO16buekdnGCNufEHPObSqqdk9o5RSRFWI9ghnnd1c6HtRtQOyvmdj23c1K7OEGbc2KO+deA1c5J7ZwO7jxoPYIZ5nVXOx/WbkDtrJjb9dzOSe3iBG3Oibns7GzrEcyonZPaOe1/a7/1CGaY113tfFi7AbWzYm7XczsntYsTtDkn5tavX289ghm1c1I7p4z/yrAewQzzuqudD2s3oHZWzO16buekdnGCNudERERERERERESMaHNOzGVlZVmPYEbtnNTOiflXX5jXXe18WLsBtbNibtdzOye1ixO0OSfmOjs7rUcwo3ZOaufkinRZj2CGed3Vzoe1G1A7K+Z2PbdzUrs4QZtzYq6iosJ6BDNq56R2TolnJVqPYIZ53dXOh7UbUDsr5nY9t3NSuzhBm3MiIiIiIiIiIiJGXH6/3289xMnA5/PB4/HA6/XC7XZbj3NCaWtrQ0REhPUYJtSudjbB2l7gKnD8PkLjQtHZ6OyvAiz2L3b09nsrWNe9L6idr521G1C72oOLntudFazr3hfUztneW0e7V6RXzom5rVu3Wo9gRu2c1M4p9ZJU6xHMMK+72vmwdgNqZ8Xcrud2TmoXJ/R6c+7DDz/EN7/5TUybNg179uwBAPz2t7/FP//5z+M2nHBoamqyHsGM2jmpnVPkgEjrEcwwr7va+bB2A2pnxdyu53ZOahcn9Gpz7k9/+hNmzZqF6OhorFu3Dq2trQAAr9eLu+6667gOKCe/uLg46xHMqJ2T2jm17mm1HsEM87qrnQ9rN6B2Vsztem7npHZxQq82537+85/jiSeewFNPPYXw8PDA10899VQUFRUdt+GEw5AhQ6xHMKN2TmrnVP3nausRzDCvu9r5sHYDamfF3K7ndk5qFyf0anNuy5YtOOOMMw77usfjQUNDw386k5ApLi62HsGM2jmpndOg6wdZj2CGed3Vzoe1G1A7K+Z2PbdzUrs4oVebc2lpadi+ffthX//nP/+JwYMH/8dDiYiIiIiIiIiIMOjV5tzVV1+NH/zgB1i1ahVcLhcqKyvxwgsv4Oabb8Y111xzvGeUk1xmZqb1CGbUzkntnOrerbMewQzzuqudD2s3oHZWzO16buekdnFCWG/+0q233oquri5Mnz4dzc3NOOOMMxAZGYmbb74Z119//fGeUU5yISG9/tDgE57aOamdk7/Tbz2CGeZ1Vzsf1m5A7ayY2/Xczknt4oRePbIulws/+clPUFdXh40bN+Kjjz5CTU0N7rzzzuM9nxDYtWuX9Qhm1M5J7Zz6f6W/9QhmmNdd7XxYuwG1s2Ju13M7J7WLE3q1Off888+jtLQUERERGDlyJKZMmYK4uDi0tLTg+eefP94zioiIiIiIiIiInJRcfr//mF+LGxISgtjYWDz77LO46KKLAl+vrq5Geno6Ojs7j+uQJwKfzwePxwOv1wu32209zgnl4MGDiI6Oth7DhNrVziZY2wtcBY7fR1hCGDrqOxy9j8X+xY7efm8F67r3BbXztbN2A2pXe3DRc7uzgnXd+4LaOdt762j3inr9C8MFBQW44oorcPvtt/f2JkQAAGVlZdYjmFE7J7VzSj4v2XoEM8zrrnY+rN2A2lkxt+u5nZPaxQm93pz75je/iXfffRdPPvkkLr74Yhw8ePB4ziVEDhw4YD2CGbVzUjunqEFR1iOYYV53tfNh7QbUzoq5Xc/tnNQuTuj1B0IAwJe+9CWsWrUK27dvxymnnIKdO3cez9mEBPPLYtXOSe2c2mrarEcww7zuaufD2g2onRVzu57bOaldnNDr95yrqqpCSkoKAKC5uRlz587FO++8g6amJr3nnN5z7pi0t7cjPDzcegwTalc7m2Bt74v3pQmJCkFXS5ej9xGs70sTrOveF9TO187aDahd7cFFz+3OCtZ17wtq52zvLUffc27x4sWIi4sL/DkmJgavvPIKfvjDH+KMM87ozU0KsaKiIusRzKidk9o5Zd2YZT2CGeZ1Vzsf1m5A7ayY2/Xczknt4oSw3vylxYuPvHtfUOD8v06IiIiIiIiIiIicLI56c+61117D7NmzER4ejtdee+0zr+dyuXD++ecfl+GEw8CBA61HMKN2TmrnVP9hvfUIZpjXXe18WLsBtbNibtdzOye1ixOOenNuzpw5gfeZmzNnzmdez+VyUb7nnPReRESE9Qhm1M5J7Zw6D/A+NzKvu9r5sHYDamfF3K7ndk5qFycc9XvOdXV1BT4Aoqur6zP/08acHKuysjLrEcyonZPaOSWdm2Q9ghnmdVc7H9ZuQO2smNv13M5J7eKEY/pAiJUrV+L111/v8bXnn38eOTk5SElJwfz589Ha2npcBxQRERERERERETlZHdPm3B133IGSkpLAnzds2IB58+ZhxowZuPXWW/HXv/4Vd99993EfUk5uY8aMsR7BjNo5qZ3Tnv/eYz2CGeZ1Vzsf1m5A7ayY2/Xczknt4oRj2pwrLi7G9OnTA3/+/e9/j6lTp+Kpp57CjTfeiIcffhh/+MMfjvuQcnIrLy+3HsGM2jmpnVPCWQnWI5hhXne182HtBtTOirldz+2c1C5OOKbNufr6eqSmpgb+/MEHH2D27NmBP0+ePFmLJcesoaHBegQzauekdk4xQ2KsRzDDvO5q58PaDaidFXO7nts5qV2ccEybc6mpqYE3AGxra0NRURG+9KUvBS4/cOAAwsPDj++EctKLjIy0HsGM2jmpnVN7fbv1CGaY113tfFi7AbWzYm7XczsntYsTXH6/33+0V77mmmvw8ccf45577sGrr76K5557DpWVlYGP033hhRfw4IMPYs2aNY4NHKx8Ph88Hg+8Xi/cbrf1OCeUrq4uhIQc0z7xSUPtamcTrO0FrgLn7yQUgMMfaL7Yv9jZO+ilYF33vqB2vnbWbkDtag8uem53VrCue19QO2d7bx3tXtExPap33nknwsLCcOaZZ+Kpp57CU089FdiYA4Cnn34aM2fO7P3UQolxM7eb2jmpnVPOLTnWI5hhXne182HtBtTOirldz+2c1C5OCDuWKyclJWH58uXwer2Ii4tDaGhoj8tffvllxMXFHdcBRURERERERERETlbHtDnXzePxHPHriYmJ/9EwwiktLc16BDNq56R2Tt6PvNYjmGFed7XzYe0G1M6KuV3P7ZzULk7QLwuLudjYWOsRzKidk9o5tVa1Wo9ghnnd1c6HtRtQOyvmdj23c1K7OCGoN+c6Ozvxs5/9DDk5OYiOjkZubi7uvPNOfPozLPx+PxYtWoQBAwYgOjoaM2bMwLZt23rcTl1dHebOnQu32434+HjMmzcPjY2NPa6zfv16nH766YiKikJmZiaWLFnSJ40C7Nixw3oEM2rnpHZOKXNSrEcww7zuaufD2g2onRVzu57bOaldnBDUm3P33HMPfv3rX+PRRx9FaWkp7rnnHixZsgSPPPJI4DpLlizBww8/jCeeeAKrVq1CbGwsZs2ahZaWlsB15s6di5KSEixbtgyvv/46li9fjvnz5wcu9/l8mDlzJrKysrB27Vrce++9uP3227F06dI+7RURERERERERES4u/6dfhhZkvvrVryI1NRW/+c1vAl+76KKLEB0djf/5n/+B3+9Heno6brrpJtx8880AAK/Xi9TUVDz77LO47LLLUFpaipEjR2LNmjXIz88HALz55ps499xzUVFRgfT0dPz617/GT37yE1RVVQU+ffbWW2/Fq6++is2bNx/VrEf78bhyuAMHDqBfv37WY5hQu9rZBGt7gavA8fuIHBiJ1j3O/vrLYv9iR2+/t4J13fuC2vnaWbsBtas9uOi53VnBuu59Qe2c7b11tHtFQf3KuVNOOQXvvPMOtm7dCgD4+OOP8c9//hOzZ88GAJSVlaGqqgozZswI/B2Px4OpU6di5cqVAICVK1ciPj4+sDEHADNmzEBISAhWrVoVuM4ZZ5wR2JgDgFmzZmHLli2or68/4mytra3w+Xw9/pPeqaqqsh7BjNo5qZ2TZ8qRP0yJAfO6q50PazegdlbM7Xpu56R2cUKvPq21r9x6663w+XwYMWIEQkND0dnZiV/84heYO3cugP87MFJTU3v8vdTU1MBlVVVVSEnp+V4AYWFhSExM7HGdnJycw26j+7KEhITDZrv77rtRUHD4v8YUFhYiNjYWEydORGlpKQ4ePIh+/fohJycH69evBwBkZWWhq6sL5eXlAIDx48dj+/btaGxsRGxsLIYNG4Z169YBADIyMhAaGopdu3YBAMaOHYudO3fC5/MhKioKo0aNwtq1awEA6enpiIqKwieffAIAGD16NCoqKtDQ0ICIiAiMHz8eq1evBnDoU1bi4uKwfft2AEBeXh6qq6tRV1eHsLAwTJo0CatXr4bf70dycjISEhICm6TDhw9HXV0dampqEBISgsmTJ6OwsBCdnZ3o378/UlJSUFpaCgAYOnQofD4fqqurAQBTp05FUVER2tvbkZCQgPT0dOzYsQN1dXXIzc1Fc3Mz9u7dCwDIz8/Hxo0b0dLSAo/Hg0GDBmHDhg0AgOzsbHR0dKCiogIAMHHiRGzevBnNzc2Ii4tDbm4uPv74YwDAoEGDAAC7d+8GAIwbNw47duxAY2MjYmJiMGLECBQVFQUe77CwMOzcuRMAMGbMGOzevRterxdRUVEYPXo0CgsLAQADBgxATExM4PfuR40ahcrKStTX1yM8PBwTJ04MbACnpqbC7XYH3g8xLy8P+/btw44dO+D1epGfn481a9agq6sLycnJSExMxJYtWwAAw4YNQ319PWpqauByuTBlyhSsXbsWHR0dSExMRGpqauDxHjJkCBobGwPH9pQpU1BcXIy2tjbEx8cjIyMDGzduBAAMHjwYLS0tqKysBABMmjQJJSUlaGlpgdvtRnZ2do9jtrOzM/B4T5gwAVu3bkVTUxPi4uIwZMgQFBcXAwAyMzMREhLS45gtKyvDgQMHEB0djby8PBQVFaG+vh7R0dGIiIhAWVlZ4PEuLy9HQ0MDIiMjMXbsWKxZsyZwzMbGxgYe75EjR6Kqqgp1dXWHPd4pKSnweDyBx3vEiBGora1FbW1t4JjtfryTkpKQlJQUeJXs0KFD4fV6sW/fvsOO2cTERKSlpWHTpk0AgNzcXDQ1NQUe78mTJ2P9+vVobW1FfHw8MjMzA8dsTk4O2trasGfPHtTX1yM7O5vyHNHU1NTjmD2ac0RJSUng8XbqHOGZ6kHi9EOfOF7+WDmSv5aMqIwotFW3Ye/v9iJrQRYAoP6DenQ2dyJpdhIAoGJpBfrP6I/owdFo39+OPU/vQfbCbABAw4oGtNe2I/mCZABASGQIEALEDo9Fx4EOlD9SjpwfH3ru8RX6cHDXQaRedOi5Z+8Le9FvfD/EjYpDV0sXdt2/C9kLs+EKd+HAxwfQVNqEtMsOfVpW1R+qEDs0Fv0m9MPq1auD8hzh8/lQV1d3TOcIABg4cOAJf46oqKgItB/tOaL7mD3RzxHdz+9Ofx/RF+cI4Oi/j9i7d29gzZ3+PmL//v0IDQ0Nmu8jdu7cibq6uj75PgIIrnNE9/HeF99HdB+zwXKOKCsrQ2NjY9D9rJFyUQoaVjRg4FUDAQA1r9UgPCkc8afEAwB23rsTA78zEOH9w3Hwk4PY/4/9yJifAQCo/XstQmNCkXDmoZ8Ddz24CwMuH4CI1Ai0VLSg5i81yLw2EzFDY9BaeeiVc059H9F9XAbbOaK+vj7w/91g/FkDcO4c0dzcjP79+wftzxqAc+eIzs5OdHR0BO3PGsH4fcSR9pOOJKh/rfX3v/89Fi5ciHvvvRejRo1CcXExFixYgPvvvx9XXnklVqxYgVNPPRWVlZUYMGBA4O9deumlcLlceOmll3DXXXfhueeeC5x8uqWkpKCgoADXXHMNZs6ciZycHDz55JOByzdt2oRRo0Zh06ZNyMvLO2y21tZWtLb+30uYfT4fMjMz9WutvVBUVISJEydaj2FC7WpnE6ztffGrL4NuGITdD+929D6C9VdfgnXd+4La+dpZuwG1qz246LndWcG67n1B7ZztvXW0v9Ya1K+cW7hwIW699VZcdtllAA7tdO/atQt33303rrzySqSlHXrVQHV1dY/NuerqaowfPx7AoR3Z7l3pbh0dHairqwv8/bS0tMBO6qdvo/uyI4mMjERkZOR/HinU/+dWOye1c3L6m/dgxrzuaufD2g2onRVzu57bOaldnBDU7znX3NyMkJCeI4aGhqKrqwvAoZdupqWl4Z133glc7vP5sGrVKkybNg0AMG3aNDQ0NAReagkA7777Lrq6ujB16tTAdZYvX4729vbAdZYtW4bhw4cf9UsQpfe6Xx7MSO2c1M4p57acL77SSYp53dXOh7UbUDsr5nY9t3NSuzghqDfnzj//fPziF7/A3/72N+zcuROvvPIK7r//flx44YUAAJfLhQULFuDnP/85XnvtNWzYsAHf+ta3kJ6ejjlz5gA49LvL55xzDq6++mqsXr0a//rXv3DdddfhsssuQ3p6OgDgG9/4BiIiIjBv3jyUlJTgpZdewkMPPYQbb7zRKl1ERE42LusBRERE5LjSc7uIHCdB/WutjzzyCH72s5/h+9//Pvbt24f09HT813/9FxYtWhS4zo9+9CM0NTVh/vz5aGhowGmnnYY333wTUVFRgeu88MILuO666zB9+nSEhITgoosuwsMPPxy43OPx4O2338a1116LSZMmISkpCYsWLcL8+fP7tJfVv39gBxO1c1I7J99a3k/1Zl53tfNh7QbUzoq5Xc/tnNQuTgjqzbl+/frhwQcfxIMPPviZ13G5XLjjjjtwxx13fOZ1EhMT8eKLL37ufY0dOxYffvhhb0eV/4DHw/sR5GrnpHZOBz85aD2CGeZ1Vzsf1m5A7ayY2/Xczknt4oSg/rVW4dD9EdSM1M5J7ZxSL0m1HsEM87qrnQ9rN6B2Vsztem7npHZxgjbnREREREREREREjGhzTsyNGDHCegQzauekdk5Vv6uyHsEM87qrnQ9rN6B2Vsztem7npHZxgjbnxFxtba31CGbUzkntnOJGx1mPYIZ53dXOh7UbUDsr5nY9t3NSuzhBm3Nijvn/4GrnpHZOcWP0DTwjtfNh7QbUzoq5Xc/tnNQuTtDmnJgLCeE9DNXOSe2cutq6rEcww7zuaufD2g2onRVzu57bOaldnODy+/1+6yFOBj6fDx6PB16vF26323ocERE5BgWuAusRjovF/sXWI4iIiAQFPbeLSDA42r0ibXuKuTVr1liPYEbtnNTOKevmLOsRzDCvu9r5sHYDamfF3K7ndk5qFydoc07MdXXxvhxc7ZzUzikkgvcpl3nd1c6HtRtQOyvmdj23c1K7OIH3bCJBIykpyXoEM2rnpHZOjRsarUcww7zuaufD2g2onRVzu57bOaldnKDNOTHH/H9wtXNSO6fGjfoGnpHa+bB2A2pnxdyu53ZOahcnaHNOzG3evNl6BDNq56R2TmmXp1mPYIZ53dXOh7UbUDsr5nY9t3NSuzhBm3MiIiIiIiIiIiJGtDkn5oYOHWo9ghm1c1I7p+qXq61HMMO87mrnw9oNqJ0Vc7ue2zmpXZygzTkx5/V6rUcwo3ZOaucUPTjaegQzzOuudj6s3YDaWTG367mdk9rFCdqcE3P79u2zHsGM2jmpnZN7ktt6BDPM6652PqzdgNpZMbfruZ2T2sUJ2pwTERHpC37rAUREROS40nO7iBwnLr/fr1PKceDz+eDxeOD1euF28/4LiojIiajAVWA9wnGx2L/YegQREZGgoOd2EQkGR7tXpFfOibmioiLrEcyonZPaOQ26YZD1CGaY113tfFi7AbWzYm7XczsntYsTtDkn5trb261HMKN2TmrnFBoXaj2CGeZ1Vzsf1m5A7ayY2/Xczknt4gRtzom5xMRE6xHMqJ2T2jk1lTZZj2CGed3Vzoe1G1A7K+Z2PbdzUrs4QZtzYi4tLc16BDNq56R2Tt7VvB89z7zuaufD2g2onRVzu57bOaldnKDNOTG3adMm6xHMqJ2T2jmlX5luPYIZ5nVXOx/WbkDtrJjb9dzOSe3iBG3OiYiIiIiIiIiIGNHmnJjLzc21HsGM2jmpndO+V/dZj2CGed3Vzoe1G1A7K+Z2PbdzUrs4QZtzYq6pifeNVNXOSe2cItMirUcww7zuaufD2g2onRVzu57bOaldnBBmPYBIVVUVsrKyrMcwoXa1s2Fu93zJg7p366zHMMG87sHYXuAq6JP7yflxDsruKnP0Phb7Fzt6+70RjGveV9SudjZ6budcd7VztjtNr5wTEREREREREREx4vL7/X7rIU4GPp8PHo8HXq8XbrfbepwTSldXF0JCOPeJ1a52NsHa3ievJAoF0OnsXQTjq4iA4F33vhCM7X31yjnWYz4Y17yvqF3twUTP7c4K1nXvC2rnbO+to90r0qMq5tavX289ghm1c1I7p4z5GdYjmGFed+Z21mOeec3Vzom5nfU8B3Cvu9rFCdqcE3Otra3WI5hROye1cwpPCLcewQzzujO3sx7zzGuudk7M7aznOYB73dUuTtDmnJiLj4+3HsGM2jmpnVPz9mbrEcwwrztzO+sxz7zmaufE3M56ngO4113t4gRtzom5zMxM6xHMqJ2T2jnVv19vPYIZ5nVnbmc95pnXXO2cmNtZz3MA97qrXZygzTkxt2HDBusRzKidk9o5DfzuQOsRzDCvO3M76zHPvOZq58TcznqeA7jXXe3iBG3OiYiIiIiIiIiIGNHmnJjLycmxHsGM2jmpnVPtG7XWI5hhXnfmdtZjnnnN1c6JuZ31PAdwr7vaxQnanBNzbW1t1iOYUTsntXMK7RdqPYIZ5nVnbmc95pnXXO2cmNtZz3MA97qrXZygzTkxt2fPHusRzKidk9o5JZyeYD2CGeZ1Z25nPeaZ11ztnJjbWc9zAPe6q12coM05ERERERERERERIy6/3++3HuJk4PP54PF44PV64Xa7rcc5obS3tyM8PNx6DBNqVzubYG0vcBU4fh8hUSHoauly9D4W+xc7evu9Fazr3heCsb0vjneA95gPxjXvK2pXezDRc7uzgnXd+4LaOdt762j3ivTKOTFXWlpqPYIZtXNSO6cBVwywHsEM87ozt7Me88xrrnZOzO2s5zmAe93VLk7Q5pyYO3jwoPUIZtTOSe2cIpIjrEcww7zuzO2sxzzzmqudE3M763kO4F53tYsTtDkn5vr162c9ghm1c1I7p5bdLdYjmGFed+Z21mOeec3Vzom5nfU8B3Cvu9rFCdqcE3M5OTnWI5hROye1c6r5W431CGaY1525nfWYZ15ztXNibmc9zwHc6652cYI258Tc+vXrrUcwo3ZOaueUeU2m9QhmmNeduZ31mGdec7VzYm5nPc8B3OuudnGCNudERERERERERESMaHNOzGVlZVmPYEbtnNTOaf+y/dYjmGFed+Z21mOeec3Vzom5nfU8B3Cvu9rFCdqcE3NdXV3WI5hROye1c3KFuqxHMMO87sztrMc885qrnRNzO+t5DuBed7WLE7Q5J+bKy8utRzCjdk5q55T45UTrEcwwrztzO+sxz7zmaufE3M56ngO4113t4gRtzomIiIiIiIiIiBjR5pyYGz9+vPUIZtTOSe2cdj+y23oEM8zrztzOeswzr7naOTG3s57nAO51V7s4Ieg35/bs2YNvfvOb6N+/P6KjozFmzBgUFhYGLvf7/Vi0aBEGDBiA6OhozJgxA9u2betxG3V1dZg7dy7cbjfi4+Mxb948NDY29rjO+vXrcfrppyMqKgqZmZlYsmRJn/QJsH37dusRzKidk9o5pX491XoEM8zrztzOeswzr7naOTG3s57nAO51V7s4Iag35+rr63HqqaciPDwcf//737Fp0yb86le/QkJCQuA6S5YswcMPP4wnnngCq1atQmxsLGbNmoWWlpbAdebOnYuSkhIsW7YMr7/+OpYvX4758+cHLvf5fJg5cyaysrKwdu1a3Hvvvbj99tuxdOnSPu1l9e8bpUzUzkntnCIHRlqPYIZ53ZnbWY955jVXOyfmdtbzHMC97moXJ4RZD/B57rnnHmRmZuKZZ54JfC0nJyfwv/1+Px588EH89Kc/xde+9jUAwPPPP4/U1FS8+uqruOyyy1BaWoo333wTa9asQX5+PgDgkUcewbnnnov77rsP6enpeOGFF9DW1oann34aERERGDVqFIqLi3H//ff32MQTZ8TGxlqPYEbtnNTOqXVvq/UIZpjXnbmd9ZhnXnO1c2JuZz3PAdzrrnZxQlC/cu61115Dfn4+LrnkEqSkpGDChAl46qmnApeXlZWhqqoKM2bMCHzN4/Fg6tSpWLlyJQBg5cqViI+PD2zMAcCMGTMQEhKCVatWBa5zxhlnICIiInCdWbNmYcuWLaivrz/ibK2trfD5fD3+k94ZNmyY9Qhm1M5J7ZyqX662HsEM87ozt7Me88xrrnZOzO2s5zmAe93VLk4I6lfOffLJJ/j1r3+NG2+8ET/+8Y+xZs0a3HDDDYiIiMCVV16JqqoqAEBqas/f9U9NTQ1cVlVVhZSUlB6Xh4WFITExscd1Pv2KvE/fZlVVVY9fo+129913o6Cg4LCvFxYWIjY2FhMnTkRpaSkOHjyIfv36IScnB+vXrwcAZGVloaurK/AxxOPHj8f27dvR2NiI2NhYDBs2DOvWrQMAZGRkIDQ0FLt27QIAjB07Fjt37oTP50NUVBRGjRqFtWvXAgDS09MRFRWFTz75BAAwevRoVFRUoKGhARERERg/fjxWr14NAEhLS0NcXFzgd8bz8vJQXV2Nuro6hIWFYdKkSVi9ejX8fj+Sk5ORkJCArVu3AgCGDx+Ouro61NTUICQkBJMnT0ZhYSE6OzvRv39/pKSkoLS0FAAwdOhQ+Hw+VFcfeuKaOnUqioqK0N7ejoSEBKSnp+Of//wnEhISkJubi+bmZuzduxcAkJ+fj40bN6KlpQUejweDBg3Chg0bAADZ2dno6OhARUUFAGDixInYvHkzmpubERcXh9zcXHz88ccAgEGDBgEAdu8+9Iat48aNw44dO9DY2IiYmBiMGDECRUVFgcc7LCwMO3fuBACMGTMGu3fvhtfrRVRUFEaPHh14z8MBAwYgJiYGO3bsAACMGjUKlZWVqK+vR3h4OCZOnBjYAE5NTYXb7Q68H2JeXh727duH7du3IykpCfn5+VizZg26urqQnJyMxMREbNmyBcChE2B9fT1qamrgcrkwZcoUrF27Fh0dHUhMTERqamrg8R4yZAgaGxsDx/aUKVNQXFyMtrY2xMfHIyMjAxs3bgQADB48GC0tLaisrAQATJo0CSUlJWhpaYHb7UZ2dnaPY7azszPweE+YMAFbt25FU1MT4uLiMGTIEBQXFwMAMjMzERIS0uOYLSsrw4EDBxAdHY28vDwUFRWhvr4eo0ePRkREBMrKygKPd3l5ORoaGhAZGYmxY8dizZo1gWM2NjY28HiPHDkSVVVVqKurO+zxTklJgcfjCTzeI0aMQG1tLWprawPHbPfjnZSUhKSkJGzevDlwzHq9Xuzbt++wYzYxMRFpaWnYtGkTACA3NxdNTU2Bx3vy5MlYv349WltbER8fj8zMzMAxm5OTg7a2NuzZswf19fWYPn065TmiqakJ+fn5x3SOKCkpCTzeTp0jPFM9SJyeCAAof6wcyV9LRlRGFNqq27D3d3uRtSALAFD/QT06mzuRNDsJAFCxtAL9Z/RH9OBotO9vx56n9yB7YTYAoGFFA9pr25F8QTIAICQyBAfWH0Ds8Fh0HOhA+SPlyPnxoeceX6EPB3cdROpFh5579r6wF/3G90PcqDh0tXRh1/27kL0wG65wFw58fABNpU1IuywNAFD1hyrEDo1Fvwn9sHr16qA8R/h8PoSEhBzTOQIABg4ceMKfIzZs2ICYmBgAR3+O6D5mnTpHpH0jDdHZ0WirbUPls5XIvvl/j9l/NaC9vh3JXz10zO55eg8STk9AzNAYdHg7UP7rcuTceuiY9a72onVPK1IuPPQ9VuXzlXDnuxE3Mg6dzZ3Y/eBujHh4BJq3N+PAugNo2taEtEv/95j9fRVi82LRb1w/+Nv92HnvTmTdmIWQqBA0ljTiQPEBDJg7AABQ/adqRGdFw53vBgCU3VWGzOszEdYvDE1bmtDY2Ngn5wjg6L+P2LFjByIjIwPHrJPfR+zfvx+hoaFB833EBx98ALfb3SffRwDBdY5YuXIlEhIS+uT7iO5jNli+jygrK0NqamrQ/ayRclEKGlY0YOBVAwEANa/VIDwpHPGnxAMAdt67EwO/MxDh/cNx8JOD2P+P/ciYnwEAqP17LUJjQpFw5qGfA3c9uAsDLh+AiNQItFS0oOYvNci8NhMxQ2NQ8cSh49qp7yO6j8tgO0fU19dj1qxZQfuzBuDcOaK5uRnjxo0L2p81AOfOEZ2dnUhISAjanzV6sx/h9PcRR9pPOhKX3+/3H9U1DURERCA/Px8rVqwIfO2GG27AmjVrsHLlSqxYsQKnnnoqKisrMWDAgMB1Lr30UrhcLrz00ku466678NxzzwVOPt1SUlJQUFCAa665BjNnzkROTg6efPLJwOWbNm3CqFGjsGnTJuTl5R02W2trK1pb/+9lzD6fD5mZmfB6vXC73cfzYTjprVq1ClOnTrUew4Ta1c4mWNsLXIf/Y8vxlvPjHJTdVebofSz2L3b09nsrWNe9LwRje18c7wDvMR+Ma95X1K72YKLndmcF67r3BbVztveWz+eDx+P5wr2ioP611gEDBmDkyJE9vpaXlxf4l8u0tEP/Atu9C9qturo6cFlaWlpgV7pbR0cH6urqelznSLfx6fv4d5GRkYF/Gez+T3onIyPDegQzauekdk5179dZj2CGed2Z21mPeeY1Vzsn5nbW8xzAve5qFycE9ebcqaeeetgr3rZu3YqsrEMvC87JyUFaWhreeeedwOU+nw+rVq3CtGnTAADTpk1DQ0ND4KWWAPDuu++iq6srsOM7bdo0LF++HO3t7YHrLFu2DMOHDz/qlyBK74WGhlqPYEbtnNTOyd8atC9UdxzzujO3sx7zzGuudk7M7aznOYB73dUuTgjqzbkf/vCH+Oijj3DXXXdh+/btePHFF7F06VJce+21AACXy4UFCxbg5z//OV577TVs2LAB3/rWt5Ceno45c+YAOPRKu3POOQdXX301Vq9ejX/961+47rrrcNlllyE9PR0A8I1vfAMRERGYN28eSkpK8NJLL+Ghhx7CjTfeaJVOpfv31xmpnZPaOfWf1d96BDPM687cznrMM6+52jkxt7Oe5wDudVe7OCGoPxBi8uTJeOWVV3DbbbfhjjvuQE5ODh588EHMnTs3cJ0f/ehHaGpqwvz589HQ0IDTTjsNb775JqKiogLXeeGFF3Dddddh+vTpCAkJwUUXXYSHH344cLnH48Hbb7+Na6+9FpMmTUJSUhIWLVqE+fPn92mviIiIiIiIiIhwCeoPhDiRHO2b/MnhDh48iOjoaOsxTKhd7WyCtb0v3jQ6vH842ve3f/EV/wPB+qbRwbrufSEY2/vqAyFYj/lgXPO+ona1BxM9tzsrWNe9L6ids723TooPhBAOO3futB7BjNo5qZ0T86++MK87czvrMc+85mrnxNzOep4DuNdd7eIEbc6JOZ/PZz2CGbVzUjun6Gzef2VkXnfmdtZjnnnN1c6JuZ31PAdwr7vaxQnanBNzn35/QDZq56R2Tm21bdYjmGFed+Z21mOeec3Vzom5nfU8B3Cvu9rFCdqcE3OjRo2yHsGM2jmpnVPls5XWI5hhXnfmdtZjnnnN1c6JuZ31PAdwr7vaxQnanBNza9eutR7BjNo5qZ1T9s3Z1iOYYV535nbWY555zdXOibmd9TwHcK+72sUJ2pwTERERERERERExos05MZeenm49ghm1c1I7p4Z/NViPYIZ53ZnbWY955jVXOyfmdtbzHMC97moXJ2hzTswxv6mk2jmpnVN7fbv1CGaY1525nfWYZ15ztXNibmc9zwHc6652cYI258TcJ598Yj2CGbVzUjun5K8mW49ghnndmdtZj3nmNVc7J+Z21vMcwL3uahcnaHNORERERERERETEiDbnxNzo0aOtRzCjdk5q57Tn6T3WI5hhXnfmdtZjnnnN1c6JuZ31PAdwr7vaxQnanBNzFRUV1iOYUTsntXNKOD3BegQzzOvO3M56zDOvudo5MbeznucA7nVXuzhBm3NirqGhwXoEM2rnpHZOMUNjrEcww7zuzO2sxzzzmqudE3M763kO4F53tYsTtDkn5iIiIqxHMKN2Tmrn1OHtsB7BDPO6M7ezHvPMa652TsztrOc5gHvd1S5OcPn9fr/1ECcDn88Hj8cDr9cLt9ttPc4Jxe/3w+VyWY9hQu1qZxOs7QWuAufvJARAl7N3sdi/2Nk76KVgXfe+EIztfXK8A7THfDCueV9Ru9qDiZ7bnRWs694X1M7Z3ltHu1ekV86JudWrV1uPYEbtnNTOKefWHOsRzDCvO3M76zHPvOZq58TcznqeA7jXXe3iBG3OiYiIiIiIiIiIGNHmnJhLS0uzHsGM2jmpnZN3tdd6BDPM687cznrMM6+52jkxt7Oe5wDudVe7OEGbc2IuLi7OegQzauekdk6te1qtRzDDvO7M7azHPPOaq50TczvreQ7gXne1ixO0OSfmtm/fbj2CGbVzUjunlAtTrEcww7zuzO2sxzzzmqudE3M763kO4F53tYsTtDknIiIiIiIiIiJiRJtzYi4vL896BDNq56R2TpXPV1qPYIZ53ZnbWY955jVXOyfmdtbzHMC97moXJ2hzTsxVV1dbj2BG7ZzUzsmd77YewQzzujO3sx7zzGuudk7M7aznOYB73dUuTtDmnJirq6uzHsGM2jmpnVPcSN430GVed+Z21mOeec3Vzom5nfU8B3Cvu9rFCdqcE3NhYWHWI5hROye1c+ps7rQewQzzujO3sx7zzGuudk7M7aznOYB73dUuTnD5/X6/9RAnA5/PB4/HA6/XC7eb9+XNIiInogJXgfUIx8Vi/2LrEeQEcLIc74COeRH5bCfLuU7nOZET29HuFemVc2Ju9erV1iOYUTsntXPKviXbegQzzOvO3M56zDOvudo5MbeznucA7nVXuzhBm3NijvnFm2rnpHZOrlCX9QhmmNeduZ31mGdec7VzYm5nPc8B3OuudnGCNufEXHJysvUIZtTOSe2cDqw7YD2CGeZ1Z25nPeaZ11ztnJjbWc9zAPe6q12coM05MZeQkGA9ghm1c1I7p6ZtTdYjmGFed+Z21mOeec3Vzom5nfU8B3Cvu9rFCdqcE3Nbt261HsGM2jmpnVPapWnWI5hhXnfmdtZjnnnN1c6JuZ31PAdwr7vaxQnanBMRERERERERETGizTkxN3z4cOsRzKidk9o5Vf2+ynoEM8zrztzOeswzr7naOTG3s57nAO51V7s4QZtzYq6urs56BDNq56R2TrF5sdYjmGFed+Z21mOeec3Vzom5nfU8B3Cvu9rFCdqcE3M1NTXWI5hROye1c+o3rp/1CGaY1525nfWYZ15ztXNibmc9zwHc6652cYI258RcSAjvYah2Tmrn5G/3W49ghnndmdtZj3nmNVc7J+Z21vMcwL3uahcnuPx+P+8Z5Tjy+XzweDzwer1wu93W44iIyDEocBVYj3BcLPYvth5BTgAny/EO6JgXkc92spzrdJ4TObEd7V6Rtj3FXGFhofUIZtTOSe2csm7Msh7BDPO6M7ezHvPMa652TsztrOc5gHvd1S5O0OacmOvs7LQewYzaOamdU0gU71Mu87ozt7Me88xrrnZOzO2s5zmAe93VLk7gPZtI0Ojfv7/1CGbUzkntnBpLGq1HMMO87sztrMc885qrnRNzO+t5DuBed7WLE7Q5J+ZSUlKsRzCjdk5q53Sg+ID1CGaY1525nfWYZ15ztXNibmc9zwHc6652cYI258RcaWmp9Qhm1M5J7ZwGzB1gPYIZ5nVnbmc95pnXXO2cmNtZz3MA97qrXZygzTkREREREREREREj2pwTc0OHDrUewYzaOamdU/Wfqq1HMMO87sztrMc885qrnRNzO+t5DuBed7WLE7Q5J+Z8Pp/1CGbUzkntnKKzoq1HMMO87sztrMc885qrnRNzO+t5DuBed7WLE7Q5J+aqq3n/xUntnNTOyZ3vth7BDPO6M7ezHvPMa652TsztrOc5gHvd1S5O0OaciIiIiIiIiIiIEZff7/dbD3Ey8Pl88Hg88Hq9cLt5/wVFROREVOAqsB7huFjsX2w9gpwATpbjHdAxLyKf7WQ51+k8J3JiO9q9Ir1yTswVFRVZj2BG7ZzUzinz+kzrEcwwrztzO+sxz7zmaufE3M56ngO4113t4gRtzom59vZ26xHMqJ2T2jmF9QuzHsEM87ozt7Me88xrrnZOzO2s5zmAe93VLk7Q5pyYS0hIsB7BjNo5qZ1T05Ym6xHMMK87czvrMc+85mrnxNzOep4DuNdd7eKEE2pz7pe//CVcLhcWLFgQ+FpLSwuuvfZa9O/fH3FxcbjooosO+wSR3bt347zzzkNMTAxSUlKwcOFCdHR09LjO+++/j4kTJyIyMhJDhgzBs88+2wdFAgDp6enWI5hROye1c2pY0WA9ghnmdWduZz3mmddc7ZyY21nPcwD3uqtdnHDCbM6tWbMGTz75JMaOHdvj6z/84Q/x17/+FS+//DI++OADVFZW4utf/3rg8s7OTpx33nloa2vDihUr8Nxzz+HZZ5/FokWLAtcpKyvDeeedh7PPPhvFxcVYsGABvvvd7+Ktt97qsz5mJSUl1iOYUTsntXMaeNVA6xHMMK87czvrMc+85mrnxNzOep4DuNdd7eKEE2JzrrGxEXPnzsVTTz3V42WUXq8Xv/nNb3D//ffjy1/+MiZNmoRnnnkGK1aswEcffQQAePvtt7Fp0yb8z//8D8aPH4/Zs2fjzjvvxGOPPYa2tjYAwBNPPIGcnBz86le/Ql5eHq677jpcfPHFeOCBB0x6RURERERERESEwwmxOXfttdfivPPOw4wZM3p8fe3atWhvb+/x9REjRmDQoEFYuXIlAGDlypUYM2YMUlNTA9eZNWsWfD5fYNd35cqVh932rFmzArdxJK2trfD5fD3+k97Jzc21HsGM2jmpnVPNazXWI5hhXnfmdtZjnnnN1c6JuZ31PAdwr7vaxQlB//Eyv//971FUVIQ1a9YcdllVVRUiIiIQHx/f4+upqamoqqoKXOfTG3Pdl3df9nnX8fl8OHjwIKKjow+777vvvhsFBQWHfb2wsBCxsbGYOHEiSktLcfDgQfTr1w85OTlYv349ACArKwtdXV0oLy8HAIwfPx7bt29HY2MjYmNjMWzYMKxbtw4AkJGRgdDQUOzatQsAMHbsWOzcuRM+nw9RUVEYNWoU1q5dC+DQ739HRUXhk08+AQCMHj0aFRUVaGhoQEREBMaPH4/Vq1cDANLS0hAXF4ft27cDAPLy8lBdXY26ujqEhYVh0qRJWL16Nfx+P5KTk5GQkICtW7cCAIYPH466ujrU1NQgJCQEkydPRmFhITo7O9G/f3+kpKSgtLQUADB06FD4fL7A+wBOnToVRUVFaG9vR0JCAtLT07FhwwbExMQgNzcXzc3N2Lt3LwAgPz8fGzduREtLCzweDwYNGoQNGzYAALKzs9HR0YGKigoAwMSJE7F582Y0NzcjLi4Oubm5+PjjjwEAgwYNAnDovQcBYNy4cdixYwcaGxsRExODESNGBD4SOiMjA2FhYdi5cycAYMyYMdi9eze8Xi+ioqIwevRoFBYWAgAGDBiAmJgY7NixAwAwatQoVFZWor6+HuHh4Zg4cSJWrVoVOJ7cbje2bdsWeLz37duH8vJy9OvXD/n5+VizZg26urqQnJyMxMREbNmyBQAwbNgw1NfXo6amBi6XC1OmTMHatWvR0dGBxMREpKamBh7vIUOGoLGxMXBsT5kyBcXFxWhra0N8fDwyMjKwceNGAMDgwYPR0tKCyspKAMCkSZNQUlKClpYWuN1uZGdn9zhmOzs7A4/3hAkTsHXrVjQ1NSEuLg5DhgxBcXExACAzMxMhISE9jtmysjIcOHAA0dHRyMvLQ1FREZqbmzF06FBERESgrKws8HiXl5ejoaEBkZGRGDt2bOD/+2lpaYiNjQ083iNHjkRVVRXq6uoOe7xTUlLg8XgCj/eIESNQW1uL2trawDHb/XgnJSUhKSkJmzdvDhyzXq8X+/btO+yYTUxMRFpaGjZt2gTg0JNTU1NT4PGePHky1q9fj9bWVsTHxyMzMzNwzObk5KCtrQ179uxBc3MzTjvtNMpzREhICCIiIo7pHNH9DylOniM8Uz1InJ4IACh/rBzJX0tGVEYU2qrbsPd3e5G1IAsAUP9BPTqbO5E0OwkAULG0Av1n9Ef04Gi072/Hnqf3IHthNoBD70PTXtuO5AuSAQCNmxoRMzwGscNj0XGgA+WPlCPnxzkAAF+hDwd3HUTqRYeei/a+sBf9xvdD3Kg4dLV0Ydf9u5C9MBuucBcOfHwATaVNSLssDQBQ9YcqxA6NRb8J/bB69eqgPEdER0cH/r97tOcIABg4cOAJf44oLy8PzHS054juY9apc0TaN9IQnR2Ntto2VD5bieyb//eY/VcD2uvbkfzVQ8fsnqf3IOH0BMQMjUGHtwPlvy5Hzq2Hjlnvai9a97Qi5cIUAEDl85Vw57sRNzIOnc2d2P3gbqR/Ox3tde04sO4AmrY1Ie3S/z1mf1+F2LxY9BvXD/52P3beuxNZN2YhJCoEjSWNOFB8AAPmDgAAVP+pGtFZ0XDnuwEA/7+9Ow+Pqj7fP35P9n3fycoSwhrWxICtUCiRUhekLnyRImj9aYMasGi1aqS2IloFlxTqBq2K1VpBBQzGCGktkkAwAooIGLIQspB9IQvJ+f2BzGXEAqGcecY89+u6uFrCMPO88zk5nvkwmRQ9WoSoO6Lg5O2ElgMtaG5utsk5Ajj/64i6ujrrmpt9HVFTUwNHR0e7uY44cOAADh8+bJPrCMC+zhGnr2dtcR1x+pi1l+uI8vJy+Pn52d1zjZBZIajfXm/91tPqd6vhHOQMvwl+AIAjTxxBvwX94BzojBNfn0DNhzWIvDUSAHD8/eNw9HCE/2WnvmureGUxwmeHwyXUBW1lbah+pxpRaVFwDnRG5ZunHtus64jTx6W9nSNaW1sxadIku32uAZh3jnB2doaDg4PdPtcAzDtHBAYGorq62m6fa1zIfoTZ1xHn+0M0LIZhGOd1SwGlpaUYN24csrOzre81N2nSJIwaNQorV67EunXrMH/+fLS3t/f4e0lJSZg8eTKWL1+OW2+9FcXFxT3eP661tRWenp7YvHkzpk+fjvj4eMyfPx/33Xef9TabN2/GjBkz0Nra+r2bc+3t7T0et7GxEVFRUWhoaICPj8/F/lT0aXl5eUhOTpYeQwTb2a6NvbYvtZz5jy0XW9z9cSh6tMjUx8gwMky9/wtlr+tuC/bYbovjHdB7zNvjmtsK29luT/jfdnPZ67rbAtt1tl+oxsZG+Pr6nnOvyK6/rbWgoABVVVUYM2YMnJyc4OTkhNzcXDzzzDNwcnJCaGgoOjo6UF9f3+PvVVZWIizs1L/OhoWFnfHTW0///ly38fHx+d6NOQBwdXWFj49Pj19ERERERERERES9Ydebc1OmTMHevXtRWFho/TVu3DjMmTPH+v+dnZ2Rk5Nj/TsHDhxASUkJUlJSAAApKSnYu3ev9WWjAJCdnQ0fHx8MHTrUeptv38fp25y+DzLXuHHjpEcQw3ad2K7TkSeOSI8gRvO6a27XesxrXnO266S5Xet5DtC97mwnM9j15py3tzeGDx/e45enpycCAwMxfPhw+Pr64uabb8bixYuxdetWFBQUYP78+UhJScEll1wCAJg2bRqGDh2KuXPn4rPPPsOWLVvwwAMPIC0tDa6urgCA2267DV9//TXuuecefPnll/jzn/+MN998E4sWLZLMV+P0+xJoxHad2K5TvwX9pEcQo3ndNbdrPeY1rznbddLcrvU8B+hed7aTGez+B0Kcy4oVK+Dg4IBZs2ahvb0dqamp+POf/2z9c0dHR2zcuBG33347UlJS4OnpiXnz5uH3v/+99TZxcXHYtGkTFi1ahKeffhqRkZF48cUXkZqaKpGkTltbm/QIYtiuE9t1cg50lh5BjOZ119yu9ZjXvOZs10lzu9bzHKB73dlOZvjBbc5t27atx+/d3NyQmZmJzMzM//p3YmJisHnz5rPe76RJk6w/kYRsy9fXV3oEMWzXie06nfj6hPQIYjSvu+Z2rce85jVnu06a27We5wDd6852MoNdf1sr6RAdHS09ghi268R2nWo+rJEeQYzmddfcrvWY17zmbNdJc7vW8xyge93ZTmbg5hyJ27t3r/QIYtiuE9t1irw1UnoEMZrXXXO71mNe85qzXSfN7VrPc4DudWc7mYGbc0REREREREREREJ+cO85R31PbGys9Ahi2K4T23U6/v5x6RG+11LLUtMfw3u0N7I+zTL1MTKMDFPv/0LxmNdH85qzXSfN7VrPc4DudWc7mYGvnCNxJ0+elB5BDNt1YrtOjh6O0iOI0dzOY14fzWvOdp00t2s9zwG6153tZAZuzpG4srIy6RHEsF0ntuvkf5m/9AhiNLfzmNdH85qzXSfN7VrPc4DudWc7mYGbc0REREREREREREK4OUfixowZIz2CGLbrxHadilcWS48gRnM7j3l9NK8523XS3K71PAfoXne2kxm4OUfivvzyS+kRxLBdJ7brFD47XHoEMZrbeczro3nN2a6T5nat5zlA97qznczAzTkS19raKj2CGLbrxHadXEJdpEcQo7mdx7w+mtec7Tppbtd6ngN0rzvbyQzcnCNxXl5e0iOIYbtObNepraxNegQxmtt5zOujec3ZrpPmdq3nOUD3urOdzMDNORI3YMAA6RHEsF0ntutU/U619AhiNLfzmNdH85qzXSfN7VrPc4DudWc7mYGbcyTus88+kx5BDNt1YrtOUWlR0iOI0dzOY14fzWvOdp00t2s9zwG6153tZAYn6QGIiIiIbG2pZalNHifu/jhkXZJl6mNkGBmm3j8RERERmYuvnCNx0dHR0iOIYbtObNepNqdWegQxbNdJa7vm8xzbddLcrvU8B+hed7aTGbg5R0REREREREREJISbcySupKREegQxbNeJ7ToFTAmQHkEM23XS2q75PMd2nTS3az3PAbrXne1kBm7OERERERERERERCeHmHIlLTEyUHkEM23Viu06lmaXSI4hhu05a2zWf59iuk+Z2rec5QPe6s53MwM05Enf48GHpEcSwXSe26xR8VbD0CGLYrpPWds3nObbrpLld63kO0L3ubCczcHOOxDU3N0uPIIbtOrFdJ7dIN+kRxLBdJ63tms9zbNdJc7vW8xyge93ZTmbg5hyJ8/DwkB5BDNt1YrtOHZUd0iOIYbtOWts1n+fYrpPmdq3nOUD3urOdzMDNORKXkJAgPYIYtuvEdp2OvX5MegQxbNdJa7vm8xzbddLcrvU8B+hed7aTGbg5R+J2794tPYIYtuvEdp1i0mOkRxDDdp20tms+z7FdJ83tWs9zgO51ZzuZwUl6ACIiIiIiIrMstSw1/THi7o9D1iVZpj5GhpFh6v0TEZEcvnKOxEVGRkqPIIbtOrFdp7rcOukRxLBdJ63tms9zmtu1Hu8A110rzevOdjIDN+dInJOT3hdwsl0ntuvU1dolPYIYtuuktV3zeU5zu9bjHeC6a6V53dlOZuDmHIk7cuSI9Ahi2K4T23UKmh4kPYIYtuuktV3zeU5zu9bjHeC6a6V53dlOZuC2JxERERGp0Ffeewzg+48RERH1JXzlHIkbMWKE9Ahi2K4T23Uqe75MegQxbNdJa7vWboDneK247jppXne2kxm4OUfiSkpKpEcQw3ad2K5T4NRA6RHEsF0nre1auwGe47Xiuuuked3ZTmbg5hyJa2hokB5BDNt1YrtO7v3dpUcQw3adtLZr7QZ4jteK666T5nVnO5mBm3Mkzs3NTXoEMWzXie06ddZ0So8ghu06aW3X2g3wHK8V110nzevOdjIDN+dI3PDhw6VHEMN2ndiu09GXj0qPIIbtOmlt19oN8ByvFdddJ83rznYyAzfnSNyuXbukRxDDdp3YrlPskljpEcSwXSet7Vq7AZ7jteK666R53dlOZuDmHBERERERERERkRBuzpG48PBw6RHEsF0ntutUv71eegQxbNdJa7vWboDneK247jppXne2kxm4OUfiPDw8pEcQw3ad2K5T53G9bxrNdp20tmvtBniO14rrrpPmdWc7mYGbcyTu8OHD0iOIYbtObNcp+Mpg6RHEsF0nre1auwGe47Xiuuuked3ZTmbg5hwREREREREREZEQbs6RuGHDhkmPIIbtOrFdp6NrjkqPIIbtOmlt19oN8ByvFdddJ83rznYyAzfnSFx5ebn0CGLYrhPbdfKb4Cc9ghi266S1XWs3wHO8Vlx3nTSvO9vJDNycI3F1dXXSI4hhu05s18lzsKf0CGLYrpPWdq3dAM/xWnHdddK87mwnM3BzjsQ5OztLjyCG7TqxXaeTTSelRxDDdp20tmvtBniO14rrrpPmdWc7mYGbcyRuzJgx0iOIYbtObNep9NlS6RHEsF0nre1auwGe47Xiuuuked3ZTmZwkh6AKC8vD8nJydJjiGA727XR3B53fxyKHi2SHkME29muidZugOd4e1z3pZalpj+GLdozjAxT7/9C2eu624Lmr3e262w3G185R0REREREREREJISbcyQuNDRUegQxbNeJ7To17mqUHkEM23XS2q61G+A5Xiu266T5653tZAZuzpE4Hx8f6RHEsF0ntut0oviE9Ahi2K6T1nat3QDP8VqxXSfNX+9sJzNwc47EHTx4UHoEMWzXie06hc7S+y+NbNdJa7vWboDneK3YrpPmr3e2kxm4OUdERERERERERCTE7jfnli1bhvHjx8Pb2xshISG4+uqrceDAgR63aWtrQ1paGgIDA+Hl5YVZs2ahsrKyx21KSkowY8YMeHh4ICQkBEuWLMHJkyd73Gbbtm0YM2YMXF1dMXDgQKxdu9bsPAIwZMgQ6RHEsF0ntut07LVj0iOIYbtOWtu1dgM8x2vFdp00f72zncxg95tzubm5SEtLw44dO5CdnY3Ozk5MmzYNLS0t1tssWrQI7733Hv7xj38gNzcX5eXluOaaa6x/3tXVhRkzZqCjowPbt2/HX//6V6xduxYPPfSQ9TZFRUWYMWMGJk+ejMLCQqSnp+OWW27Bli1bbNqrUVVVlfQIYtiuE9t18h7lLT2CGLbrpLVdazfAc7xWbNdJ89c728kMdr85l5WVhZtuugnDhg1DYmIi1q5di5KSEhQUFAAAGhoa8NJLL+Gpp57CT37yE4wdOxZr1qzB9u3bsWPHDgDABx98gC+++AKvvvoqRo0ahenTp+ORRx5BZmYmOjo6AACrV69GXFwcnnzySQwZMgQLFy7EL37xC6xYsUKsXYuamhrpEcSwXSe26+Q1zEt6BDFs10lru9ZugOd4rdiuk+avd7aTGex+c+67GhoaAAABAQEAgIKCAnR2dmLq1KnW2yQkJCA6OhqffPIJAOCTTz7BiBEjevzY39TUVDQ2NuLzzz+33ubb93H6Nqfv47va29vR2NjY4xddGEdHR+kRxLBdJ7br1N3WLT2CGLbrpLVdazfAc7xWbNdJ89c728kMTtID9EZ3dzfS09MxceJEDB8+HABQUVEBFxcX+Pn59bhtaGgoKioqrLf59sbc6T8//Wdnu01jYyNOnDgBd3f3Hn+2bNkyLF269IwZd+3aBU9PT4wZMwb79+/HiRMn4O3tjbi4OOzZswcAEBMTg+7ubpSWlgIARo0ahUOHDqG5uRmenp6Ij4/Hp59+CgCIjIyEo6MjiouLAQAjR47EkSNH0NjYCDc3NwwbNsz6KsKIiAi4ubnh66+/BgAMHz4cZWVlqK+vh4uLC0aNGoX8/HwAQFhYGLy8vHDo0CEAp753vLKyErW1tXBycsLYsWORn58PwzAQHBwMf39/fPXVVwCAwYMHo7a2FtXV1XBwcMD48eOxa9cudHV1ITAwECEhIdi/fz8AYNCgQWhsbLS+B2BycjJ2796Nzs5O+Pv7IyIiAl1dXcjLy8OAAQPQ2tqKY8dOvXfDuHHjsG/fPrS1tcHX1xfR0dHYu3cvACA2NhYnT55EWVkZAGDMmDH48ssv0draCi8vLwwYMACfffYZACA6OhrAqfcdBIDExEQcPnwYzc3N8PDwQEJCAnbv3m39fDs5OeHIkSMAgBEjRqCkpAQNDQ1wc3PD8OHDsWvXLgBAeHg4PDw8cPjwYQDAsGHDUF5ejrq6Ojg7O2PMmDHIy8uzHks+Pj7Wn24zZMgQVFVVoaurC7t27cK4ceOwc+dOdHd3Izg4GAEBAdb3VoyPj0ddXR2qq6thsViQlJSEgoICnDx5EgEBAQgNDbV+vgcOHIjm5mbrcZ2UlITCwkJ0dHTAz88PkZGR2LdvHwCgf//+aGtrQ3l5OQBg7Nix+Pzzz9HW1gYfHx/Exsb2OGa7urqsn+/Ro0fjq6++QktLC7y8vDBw4EAUFhYCAKKiouDg4NDjmC0qKkJTUxPc3d0xZMgQ6+e7rKwMLi4uKCoqsn6+S0tLUV9fD1dXV4wcORI7d+60HrOenp7Wz/fQoUNRUVGB2traMz7fISEh8PX1tX6+ExIScPz4cRw/ftx6zJ7+fAcFBSEoKAhffvml9ZhtaGiwvmT728dsQEAAwsLC8MUXXwAABgwYgJaWFuvne/z48dizZw/a29vh5+eHqKgo6zEbFxeHjo4OHD16FADQ2dmp9hzR2NjYq3PE6X9EMfMc4Zvsi4App/7RpzSzFMFXBcMt0g0dlR049voxxKTHAADqcuvQ1dqFoOlBp47h58sQODUQ7v3d0VnTiaMvH0XsklgAQP32enQe70TwlcEAgKNrjiJkVgg8B3viZNNJlD5birj74wAAjbsacaL4hPWnvh177Ri8R3nDa5gXutu6UfxUMWKXxMLibEHTZ01o2d+CsBvCAAAVb1bAc5AnvEd7Iz8/v9fniKi0KDj5OqH1YCvq/l2Hfgv6AQCqN1bD2d8ZfhP9AABH/nQEETdFwCXIBSeOnEDNlhpE/r9IAEDNlhpYXC0ImHTqc1jyTAlCrw2Fa7gr2o+2o+yFMmtr7Ue1MLoMBP408NTne1UpgmcEwy3aDR3VHTj2yjHELP7m8/3vOnQ1dSHoZ6c+30dfPAr/Sf7wGOiBzrpOlD1fhrh7T91vcXFxr84RHoM8EHrtN9cCr1fAa7gXvEZ4obujG8V/KkbMb2Lg4OKA5r3NaN7XjLDZpz7flf+ohHt/d/iM9QEMoGhZEaLvjIajlyNa9regIb8BEfMiAABVG6rQVNhkbS9aXoTIWyPh7O+M1kOtqNtWh363nPp8H998HI7ejvD/kf+pnqeKET43HC7BLmgraUP1pmpE3R516vOdXQOLowUBPzn1+W5vb+/VOSLs/8LgHuuOjuMdKF9bjtjffHPM/qcenXWdCP75N8fsy0fh/yN/eAzywMmGkyhdVYq4355qachvQPvRdoTMDAEAlP+tHD7jfOA11AtdrV0oWVkCi7MFcffHoenTJrQcbEHYdd8cs3+vgOcQT3gnesPoNHDkiSOIWRwDBzcHNH/ejKbCJoTPCT/1+f5nJdxj3OEzzufU5/DRIkTdEQUnbye0HGhBc3Nzr84Rp9fi+PvH4ejhCP/Lvvl8ryxG+OxwuIS6oK2sDdXvVCMq7dTnuzanFgDO+xxRu7XW+ji9PUf4TfA773PEoUOHUFNTA0dHR7u5jnB2dkZeXp7NriP69et3XtcRcffHoWFHA9or2hFy9TfH7F/L4ZvkC88hnuhq7kLJMyWIuy8OsACNBY048fWJXp0jHNwcEHd/XK/PEa5hrvC9xPe8zhFlZWXW64jzfa4Rd38cSp4tQeg1oXDt54r2Y+2o/Eclou88dX1cu60WRruBwNRT5+Syv5QhMDWwV+cIBzcHRKVF9focEXtvLCyOlvM6R1zIc42QWSGo316PfvO/+e/au9VwDnKG3wQ/AMCRJ46g34J+cA50xomvT6DmwxpE3hrZ63OEb7Jvr84Rvb2OOP3frgt5rmH2OcIwDLt+rnG+5wig9881amtr7fq5hpn7Efv377fr5xq93Y8w+7mGv/+p88i5WAzDMM7rlnbg9ttvx/vvv4+PP/4YkZGnTpzr1q3D/Pnz0d7e3uO2SUlJmDx5MpYvX45bb70VxcXFPd4/rrW1FZ6enti8eTOmT5+O+Ph4zJ8/H/fdd5/1Nps3b8aMGTPQ2tp6xuZce3t7j8dsbGxEVFQUGhoa4OPjY0Z+n7Vz506MHz9eegwRbGe7NvbavtRy5j+2XGyxS2Jx5Ikjpj5GhpHR67+jtd0W3QDbNbbbohu4sK93s/Ecf8TUx+A5vnc0t9uCvX692wLbdbZfqMbGRvj6+p5zr+gH88q5hQsXYuPGjfjXv/5l3ZgDTu24dnR0oL6+vser5yorKxEWFma9zekd2m//+ek/O/2/3/0Jr5WVlfDx8TljYw4AXF1d4erqelHatOvu1vtycLbrxHadLM4W6RHEsF0nre1auwGe47Viu06av97ZTmaw+/ecMwwDCxcuxPr16/HRRx8hLi6ux5+ffgl9Tk6O9WMHDhxASUkJUlJSAAApKSnYu3dvj58skp2dDR8fHwwdOtR6m2/fx+nbnL4PMk9wcLD0CGLYrhPbdWr6rEl6BDFs10lru9ZugOd4rdiuk+avd7aTGez+lXNpaWlYt24d3nnnHXh7e1u/39rX1xfu7u7w9fXFzTffjMWLFyMgIAA+Pj644447kJKSgksuuQQAMG3aNAwdOhRz587F448/joqKCjzwwANIS0uzvvrttttuw3PPPYd77rkHCxYswEcffYQ333wTmzZtEmvX4vQP99CI7TqxXaeW/S3SI4hhu05a27V2AzzHa8V2nez1690W387s3t8dJ74+Yepj2Ou3M9vruvcFdv/KuVWrVqGhoQGTJk1CeHi49dcbb7xhvc2KFSvw85//HLNmzcKPf/xjhIWF4e2337b+uaOjIzZu3AhHR0ekpKTgxhtvxC9/+Uv8/ve/t94mLi4OmzZtQnZ2NhITE/Hkk0/ixRdfRGpqqk17NTr9ZqQasV0ntut0+gc4aMR2nbS2a+0GeI7Xiu068etdJ83rbja7f+Xc+fy8Cjc3N2RmZiIzM/O/3iYmJgabN28+6/1MmjTJ+lNJiIiIiIiIiIiIzGb3r5yjvi8+Pl56BDFs14ntOlW8WSE9ghi266S1XWs3wHO8VmzXiV/vOmled7PZ/SvnqO+rq6uDv7+/9Bgi2M52bTS3ew7yxIlD5r4/ib1iO9s1sdduW7wPU9D0IBx//7jpj2OP78Vkr+tuC2zX2c5rOq47XVx85RyJq66ulh5BDNt1YrtO3qO9pUcQw3adtLZr7QbYrhXbdeI1nU6a191s3JwjcRaLRXoEMWzXie06GV3nfg/VvortOmlt19oNsF0rtuvEazqdNK+72bg5R+KSkpKkRxDDdp3YrtOR5UekRxDDdp20tmvtBtiuFdt14jWdTprX3WzcnCNxBQUF0iOIYbtObNcpOj1aegQxbNdJa7vWboDtWrFdJ17T6aR53c3GzTkSd/LkSekRxLBdJ7br5OjhKD2CGLbrpLVdazfAdq3YrhOv6XTSvO5m409rJXEBAQHSI4ix13Zb/ES34KuDkbUhy9THsMef5gbY77rbgub25i+apUcQw3adtLZr7QbYrhXbdeI1nU6a191sfOUciQsNDZUeQYzm9sZdjdIjiNG87prbNR/zbNdJa7vWboDtWrFdJ17T6aR53c3GzTkSt3//fukRxGhuj/hlhPQIYjSvu+Z2zcc823XS2q61G2C7VmzXidd0Omled7Nxc46IiIiIiIiIiEgIN+dI3MCBA6VHEKO5vWp9lfQIYjSvu+Z2zcc823XS2q61G2C7VmzXidd0Omled7Nxc47ENTfrfUNNze2u/VylRxCjed01t2s+5tmuk9Z2rd0A27Viu068ptNJ87qbjZtzJK6iokJ6BDGa232TfKVHEKN53TW3az7m2a6T1nat3QDbtWK7Trym00nzupuNm3NERERERERERERCuDlH4pKSkqRHEKO5veixIukRxGhed83tmo95tuuktV1rN8B2rdiuE6/pdNK87mbj5hyJKywslB5BjOb2qNujpEcQo3ndNbdrPubZrpPWdq3dANu1YrtOvKbTSfO6m42bcySuo6NDegQxmtudfJ2kRxCjed01t2s+5tmuk9Z2rd0A27Viu068ptNJ87qbjZtzJM7Pz096BDGa21sPtkqPIEbzumtu13zMs10nre1auwG2a8V2nXhNp5PmdTcbN+dIXGRkpPQIYjS31/27TnoEMZrXXXO75mOe7TppbdfaDbBdK7brxGs6nTSvu9m4OUfi9u3bJz2CGM3t/Rb0kx5BjOZ119yu+Zhnu05a27V2A2zXiu068ZpOJ83rbjZuzhEREREREREREQnh5hyJ69+/v/QIYjS3V2+slh5BjOZ119yu+Zhnu05a27V2A2zXiu068ZpOJ83rbja9P2aE7EZbW5v0CN9rqWWp6Y/hf5k/6nLNfc+CDCPD1Pu/UM7+ztIjiLHXY94WNLdrPubZrpPWdq3dANu1YrtOvKbTSfO6m42vnCNx5eXl0iOI8ZvoJz2CGM3tmo95ze2aj3m266S1XWs3wHat2K4Tr+l00rzuZuPmHBERERERERERkRBuzpG4sWPHSo8g5sifjkiPIEZzu+ZjXnO75mOe7TppbdfaDbBdK7brxGs6nTSvu9m4OUfiPv/8c+kRxETcFCE9ghjN7ZqPec3tmo95tuuktV1rN8B2rdiuE6/pdNK87mbj5hyJ0/ymki5BLtIjiNHcrvmY19yu+Zhnu05a27V2A2zXiu068ZpOJ83rbjb+tFYS5+PjIz2CmBNHTkiPIMZe223xU3rD/i8MWeuyTH0Me/0pvfx614ntOmlt19oNsF0rtuvEazqdNK+72fjKORIXGxsrPYKYmi010iOIYbtO/HrXie06aW3X2g2wXSu268RrOp00r7vZuDlH4vbs2SM9gpjI/xcpPYIYtuvEr3ed2K6T1nat3QDbtWK7Trym00nzupuNm3NERERERERERERC+J5zJC4mJkZ6BDGaXxLNdvtji/fb8xnrg6wCne+3Z6/rbgts10lru9ZugO1asV0nPofTSfO6m42vnCNxXV1d0iOIsbhapEcQw3ad2K4T23XS2q61G2C7VmzXic/hdNK87mbj5hyJKysrkx5BTMCkAOkRxLBdJ7brxHadtLZr7QbYrhXbdeJzOJ00r7vZuDlHREREREREREQkhJtzJG706NHSI4gpeaZEegQxbNeJ7TqxXSet7Vq7AbZrxXad+BxOJ83rbjZuzpG4r776SnoEMaHXhkqPIIbtOrFdJ7brpLVdazfAdq3YrhOfw+mked3Nxs05EtfS0iI9ghjXcFfpEcSwXSe268R2nbS2a+0G2K4V23XiczidNK+72bg5R+K8vLykRxDTfrRdegQxbNeJ7TqxXSet7Vq7AbZrxXad+BxOJ83rbjYn6QGIBg4cKD2CmMq3K6VHEMN2ndiuE9t10tqutRtgu1Zstz9LLUtNfwxHb0e81fSWqY+RYWSYev8Xyl7X3RY0P3c3G185R+IKCwulRxATfUe09Ahi2K4T23Viu05a27V2A2zXiu06sV0nzc/dzcbNOSIiIiIiIiIiIiHcnCNxUVFR0iOIqf2oVnoEMWzXie06sV0nre1auwG2a8V2ndiuk+bn7mbj5hyJc3DQexgaXYb0CGLYrhPbdWK7TlrbtXYDbNeK7TqxXSfNz93Nxs8siSsuLpYeQUzgTwOlRxDDdp3YrhPbddLarrUbYLtWbNeJ7Tppfu5uNm7OERERERERERERCeHmHIkbOXKk9AhiSleVSo8ghu06sV0ntuuktV1rN8B2rdiuE9t10vzc3WzcnCNxRUVF0iOICZ4RLD2CGLbrxHad2K6T1nat3QDbtWK7TmzXSfNzd7M5SQ9A1NTUJD2CGLdoN+kRxLBdJ7brxHadtLZr7QbYrhXbdWK7/VlqWWr6Y8TdH4d/PPoPUx8jw8gw9f7tFV859x2ZmZmIjY2Fm5sbkpOTkZ+fLz1Sn+fu7i49gpiO6g7pEcSwXSe268R2nbS2a+0G2K4V23Viu06a283GV859yxtvvIHFixdj9erVSE5OxsqVK5GamooDBw4gJCREejwRtth9d3BzwPq29aY+hr3uvh975Zj0CGLYrhPbdWK7TlrbtXYDbNeK7TqxXSfN7WbjK+e+5amnnsKvfvUrzJ8/H0OHDsXq1avh4eGBl19+WXq0Pi1mcYz0CGLYrhPbdWK7TmzXR2s3wHat2K4T23XS3G42vnLuGx0dHSgoKMB9991n/ZiDgwOmTp2KTz755Izbt7e3o7293fr7hoYGAEBjY6P5w9pQG9pMf4wT7SdMf5wLWRe2m4vtbDcL23tHa7stugG2a2y3RTfA9t7oK+08x/cO283Fdrabpa/tqZzuMQzjrLezGOe6hRLl5eXo168ftm/fjpSUFOvH77nnHuTm5iIvL6/H7R9++GEsXWr+t3wSEREREREREdEPV2lpKSIjI//rn/OVcxfovvvuw+LFi62/7+7uRm1tLQIDA2GxWAQn+2FpbGxEVFQUSktL4ePjIz2OTbGd7WzXg+1sZ7sOWrsBtrOd7Zqwne3a2v8XhmGgqakJERERZ70dN+e+ERQUBEdHR1RWVvb4eGVlJcLCws64vaurK1xdXXt8zM/Pz8wR+zQfHx+1X+BsZ7s2bGe7NmzX1661G2A72/VhO9u10dx+oXx9fc95G/5AiG+4uLhg7NixyMnJsX6su7sbOTk5Pb7NlYiIiIiIiIiI6GLhK+e+ZfHixZg3bx7GjRuHpKQkrFy5Ei0tLZg/f770aERERERERERE1Adxc+5brr/+elRXV+Ohhx5CRUUFRo0ahaysLISGhkqP1me5uroiIyPjjG8R1oDtbNeG7WzXhu362rV2A2xnO9s1YTvb6eLjT2slIiIiIiIiIiISwvecIyIiIiIiIiIiEsLNOSIiIiIiIiIiIiHcnCMiIiIiIiIiIhLCzTkiIiIiIiIiIiIh3JwjUZmZmYiNjYWbmxuSk5ORn58vPZLp/vWvf+GKK65AREQELBYLNmzYID2SzSxbtgzjx4+Ht7c3QkJCcPXVV+PAgQPSY9nEqlWrMHLkSPj4+MDHxwcpKSl4//33pceyucceewwWiwXp6enSo5ju4YcfhsVi6fErISFBeiybOXr0KG688UYEBgbC3d0dI0aMwK5du6THMl1sbOwZ626xWJCWliY9mum6urrw4IMPIi4uDu7u7hgwYAAeeeQRaPnZY01NTUhPT0dMTAzc3d0xYcIE7Ny5U3qsi+5c1zGGYeChhx5CeHg43N3dMXXqVBw8eFBm2IvsXO1vv/02pk2bhsDAQFgsFhQWForMaYaztXd2duLee+/FiBEj4OnpiYiICPzyl79EeXm53MAX0bnW/eGHH0ZCQgI8PT3h7++PqVOnIi8vT2bYi6w3z1tuu+02WCwWrFy50mbzmelc7TfddNMZ/62//PLLZYa9iM5nzffv348rr7wSvr6+8PT0xPjx41FSUmL7YfsYbs6RmDfeeAOLFy9GRkYGdu/ejcTERKSmpqKqqkp6NFO1tLQgMTERmZmZ0qPYXG5uLtLS0rBjxw5kZ2ejs7MT06ZNQ0tLi/RopouMjMRjjz2GgoIC7Nq1Cz/5yU9w1VVX4fPPP5cezWZ27tyJv/zlLxg5cqT0KDYzbNgwHDt2zPrr448/lh7JJurq6jBx4kQ4Ozvj/fffxxdffIEnn3wS/v7+0qOZbufOnT3WPDs7GwBw7bXXCk9mvuXLl2PVqlV47rnnsH//fixfvhyPP/44nn32WenRbOKWW25BdnY2XnnlFezduxfTpk3D1KlTcfToUenRLqpzXcc8/vjjeOaZZ7B69Wrk5eXB09MTqampaGtrs/GkF9+52ltaWnDppZdi+fLlNp7MfGdrb21txe7du/Hggw9i9+7dePvtt3HgwAFceeWVApNefOda9/j4eDz33HPYu3cvPv74Y8TGxmLatGmorq628aQX3/k+b1m/fj127NiBiIgIG01mvvNpv/zyy3v8N//111+34YTmOFf34cOHcemllyIhIQHbtm3Dnj178OCDD8LNzc3Gk/ZBBpGQpKQkIy0tzfr7rq4uIyIiwli2bJngVLYFwFi/fr30GGKqqqoMAEZubq70KCL8/f2NF198UXoMm2hqajIGDRpkZGdnG5dddplx1113SY9kuoyMDCMxMVF6DBH33nuvcemll0qPYRfuuusuY8CAAUZ3d7f0KKabMWOGsWDBgh4fu+aaa4w5c+YITWQ7ra2thqOjo7Fx48YeHx8zZozxu9/9Tmgq8333Oqa7u9sICwsznnjiCevH6uvrDVdXV+P1118XmNA8Z7uGKyoqMgAYn376qU1nspXzuX7Nz883ABjFxcW2GcpGzqe9oaHBAGB8+OGHthnKRv5be1lZmdGvXz9j3759RkxMjLFixQqbz2a272ufN2+ecdVVV4nMYyvf13399dcbN954o8xAfRxfOUciOjo6UFBQgKlTp1o/5uDggKlTp+KTTz4RnIxsqaGhAQAQEBAgPIltdXV14e9//ztaWlqQkpIiPY5NpKWlYcaMGT2+5jU4ePAgIiIi0L9/f8yZM0fNS/7fffddjBs3Dtdeey1CQkIwevRovPDCC9Jj2VxHRwdeffVVLFiwABaLRXoc002YMAE5OTn46quvAACfffYZPv74Y0yfPl14MvOdPHkSXV1dZ7xywN3dXc0rZgGgqKgIFRUVPc71vr6+SE5O5vWdMg0NDbBYLPDz85MexaY6Ojrw/PPPw9fXF4mJidLjmK67uxtz587FkiVLMGzYMOlxbG7btm0ICQnB4MGDcfvtt6OmpkZ6JFN1d3dj06ZNiI+PR2pqKkJCQpCcnKzqbZrMxM05EnH8+HF0dXUhNDS0x8dDQ0NRUVEhNBXZUnd3N9LT0zFx4kQMHz5cehyb2Lt3L7y8vODq6orbbrsN69evx9ChQ6XHMt3f//537N69G8uWLZMexaaSk5Oxdu1aZGVlYdWqVSgqKsKPfvQjNDU1SY9muq+//hqrVq3CoEGDsGXLFtx+++2488478de//lV6NJvasGED6uvrcdNNN0mPYhO//e1vccMNNyAhIQHOzs4YPXo00tPTMWfOHOnRTOft7Y2UlBQ88sgjKC8vR1dXF1599VV88sknOHbsmPR4NnP6Go7Xd7q1tbXh3nvvxezZs+Hj4yM9jk1s3LgRXl5ecHNzw4oVK5CdnY2goCDpsUy3fPlyODk54c4775QexeYuv/xy/O1vf0NOTg6WL1+O3NxcTJ8+HV1dXdKjmaaqqgrNzc147LHHcPnll+ODDz7AzJkzcc011yA3N1d6vB88J+kBiEintLQ07Nu3T9UrCgYPHozCwkI0NDTgrbfewrx585Cbm9unN+hKS0tx1113ITs7W917UXz71UIjR45EcnIyYmJi8Oabb+Lmm28WnMx83d3dGDduHB599FEAwOjRo7Fv3z6sXr0a8+bNE57Odl566SVMnz69T70Hz9m8+eabeO2117Bu3ToMGzYMhYWFSE9PR0REhIp1f+WVV7BgwQL069cPjo6OGDNmDGbPno2CggLp0YhsprOzE9dddx0Mw8CqVaukx7GZyZMno7CwEMePH8cLL7yA6667Dnl5eQgJCZEezTQFBQV4+umnsXv3bhWvDv+uG264wfr/R4wYgZEjR2LAgAHYtm0bpkyZIjiZebq7uwEAV111FRYtWgQAGDVqFLZv347Vq1fjsssukxzvB4+vnCMRQUFBcHR0RGVlZY+PV1ZWIiwsTGgqspWFCxdi48aN2Lp1KyIjI6XHsRkXFxcMHDgQY8eOxbJly5CYmIinn35aeixTFRQUoKqqCmPGjIGTkxOcnJyQm5uLZ555Bk5OTn36Xxe/y8/PD/Hx8Th06JD0KKYLDw8/Y9N5yJAhar6tFwCKi4vx4Ycf4pZbbpEexWaWLFliffXciBEjMHfuXCxatEjNq2YHDBiA3NxcNDc3o7S0FPn5+ejs7ET//v2lR7OZ09dwvL7T6fTGXHFxMbKzs9W8ag4APD09MXDgQFxyySV46aWX4OTkhJdeekl6LFP9+9//RlVVFaKjo63XeMXFxbj77rsRGxsrPZ7N9e/fH0FBQX36Oi8oKAhOTk7qr/HMws05EuHi4oKxY8ciJyfH+rHu7m7k5OSoeQ8ujQzDwMKFC7F+/Xp89NFHiIuLkx5JVHd3N9rb26XHMNWUKVOwd+9eFBYWWn+NGzcOc+bMQWFhIRwdHaVHtJnm5mYcPnwY4eHh0qOYbuLEiThw4ECPj3311VeIiYkRmsj21qxZg5CQEMyYMUN6FJtpbW2Fg0PPS0tHR0frv7Rr4enpifDwcNTV1WHLli246qqrpEeymbi4OISFhfW4vmtsbEReXh6v7/q40xtzBw8exIcffojAwEDpkURpuMabO3cu9uzZ0+MaLyIiAkuWLMGWLVukx7O5srIy1NTU9OnrPBcXF4wfP179NZ5Z+G2tJGbx4sWYN28exo0bh6SkJKxcuRItLS2YP3++9Gimam5u7vEvKkVFRSgsLERAQACio6MFJzNfWloa1q1bh3feeQfe3t7W95/x9fWFu7u78HTmuu+++zB9+nRER0ejqakJ69atw7Zt2/r8xYu3t/cZ7yno6emJwMDAPv9eg7/5zW9wxRVXICYmBuXl5cjIyICjoyNmz54tPZrpFi1ahAkTJuDRRx/Fddddh/z8fDz//PN4/vnnpUezie7ubqxZswbz5s2Dk5OeS60rrrgCf/zjHxEdHY1hw4bh008/xVNPPYUFCxZIj2YTW7ZsgWEYGDx4MA4dOoQlS5YgISGhz13XnOs6Jj09HX/4wx8waNAgxMXF4cEHH0RERASuvvpquaEvknO119bWoqSkBOXl5QBgfQIbFhb2g3/l4Nnaw8PD8Ytf/AK7d+/Gxo0b0dXVZb3GCwgIgIuLi9TYF8XZ2gMDA/HHP/4RV155JcLDw3H8+HFkZmbi6NGjuPbaawWnvjjOdcx/dxPW2dkZYWFhGDx4sK1HvejO1h4QEIClS5di1qxZCAsLw+HDh3HPPfdg4MCBSE1NFZz6f3euNV+yZAmuv/56/PjHP8bkyZORlZWF9957D9u2bZMbuq8Q/mmxpNyzzz5rREdHGy4uLkZSUpKxY8cO6ZFMt3XrVgPAGb/mzZsnPZrpvq8bgLFmzRrp0Uy3YMECIyYmxnBxcTGCg4ONKVOmGB988IH0WCIuu+wy46677pIew3TXX3+9ER4ebri4uBj9+vUzrr/+euPQoUPSY9nMe++9ZwwfPtxwdXU1EhISjOeff156JJvZsmWLAcA4cOCA9Cg21djYaNx1111GdHS04ebmZvTv39/43e9+Z7S3t0uPZhNvvPGG0b9/f8PFxcUICwsz0tLSjPr6eumxLrpzXcd0d3cbDz74oBEaGmq4uroaU6ZM6TNfC+dqX7Nmzff+eUZGhujcF8PZ2ouKiv7rNd7WrVulR/+fna39xIkTxsyZM42IiAjDxcXFCA8PN6688kojPz9feuyLorfPW2JiYowVK1bYdEaznK29tbXVmDZtmhEcHGw4OzsbMTExxq9+9SujoqJCeuz/2fms+UsvvWQMHDjQcHNzMxITE40NGzbIDdyHWAzDMC7WRh8RERERERERERGdP77nHBERERERERERkRBuzhEREREREREREQnh5hwREREREREREZEQbs4REREREREREREJ4eYcERERERERERGREG7OERERERERERERCeHmHBERERERERERkRBuzhEREREREREREQnh5hwRERERwWKxYMOGDaY/zqRJk5Cenn7et9+2bRssFgvq6+v/623Wrl0LPz+//3k2IiIiIgncnCMiIiLq4yoqKnDHHXegf//+cHV1RVRUFK644grk5ORIj3ZOEyZMwLFjx+Dr6ys9ChEREZEpnKQHICIiIiLzHDlyBBMnToSfnx+eeOIJjBgxAp2dndiyZQvS0tLw5ZdfSo94Vi4uLggLC5Meg4iIiMg0fOUcERERUR/261//GhaLBfn5+Zg1axbi4+MxbNgwLF68GDt27Ohx2+PHj2PmzJnw8PDAoEGD8O6771r/7Pu+dXTDhg2wWCzW3z/88MMYNWoUXnnlFcTGxsLX1xc33HADmpqa/ut8mzZtgq+vL1577bXv/fPv+7bWtWvXIjo6Gh4eHpg5cyZqamp68RkhIiIisi/cnCMiIiLqo2pra5GVlYW0tDR4enqe8eff3WxbunQprrvuOuzZswc/+9nPMGfOHNTW1vbqMQ8fPowNGzZg48aN2LhxI3Jzc/HYY499723XrVuH2bNn47XXXsOcOXPO6/7z8vJw8803Y+HChSgsLMTkyZPxhz/8oVczEhEREdkTbs4RERER9VGHDh2CYRhISEg4r9vfdNNNmD17NgYOHIhHH30Uzc3NyM/P79Vjdnd3Y+3atRg+fDh+9KMfYe7cud/73naZmZn49a9/jffeew8///nPz/v+n376aVx++eW45557EB8fjzvvvBOpqam9mpGIiIjInvA954iIiIj6KMMwenX7kSNHWv+/p6cnfHx8UFVV1av7iI2Nhbe3t/X34eHhZ9zHW2+9haqqKvznP//B+PHje3X/+/fvx8yZM3t8LCUlBVlZWb26HyIiIiJ7wVfOEREREfVRgwYNgsViOe8f+uDs7Nzj9xaLBd3d3QAABweHMzb7Ojs7e3Ufp40ePRrBwcF4+eWXe72BSERERNTXcHOOiIiIqI8KCAhAamoqMjMz0dLScsaff/uHLJxLcHAwmpqaetxPYWHhBc01YMAAbN26Fe+88w7uuOOOXv3dIUOGIC8vr8fHvvuDLYiIiIh+SLg5R0RERNSHZWZmoqurC0lJSfjnP/+JgwcPYv/+/XjmmWeQkpJy3veTnJwMDw8P3H///Th8+DDWrVuHtWvXXvBc8fHx2Lp1K/75z38iPT39vP/enXfeiaysLPzpT3/CwYMH8dxzz/FbWomIiOgHjZtzRERERH1Y//79sXv3bkyePBl33303hg8fjp/+9KfIycnBqlWrzvt+AgIC8Oqrr2Lz5s0YMWIEXn/9dTz88MP/02yDBw/GRx99hNdffx133333ef2dSy65BC+88AKefvppJCYm4oMPPsADDzzwP81BREREJMli8I0+iIiIiIiIiIiIRPCVc0REREREREREREK4OUdERERERERERCSEm3NERERERERERERCuDlHREREREREREQkhJtzREREREREREREQrg5R0REREREREREJISbc0REREREREREREK4OUdERERERERERCSEm3NERERERERERERCuDlHREREREREREQkhJtzREREREREREREQv4/xeuXq3NbBTwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1500x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = range(len(chunk_size_list))\n",
    "bar_color = 'purple'\n",
    "\n",
    "plt.figure(figsize=(15, 7))\n",
    "plt.bar(x, chunk_size_list, color=bar_color)\n",
    "plt.xlabel('Chunk id')\n",
    "plt.ylabel('Size')\n",
    "plt.title('Size of Semantic chunks')\n",
    "plt.xticks(x)\n",
    "plt.grid(True, which='both', axis='both', linestyle='--', linewidth=0.5)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8da96cab",
   "metadata": {},
   "source": [
    "## 3. Azure AI Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "21a75cf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "azure_openai_embeddings = AzureOpenAIEmbeddings(\n",
    "    azure_deployment=AOAI_EMBEDDINGS_MODEL,\n",
    "    openai_api_version=AOAI_API_VERSION,\n",
    "    azure_endpoint=AOAI_API_ENDPOINT,\n",
    "    openai_api_type=AOAI_API_TYPE,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4bc784ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AzureOpenAIEmbeddings(client=<openai.resources.embeddings.Embeddings object at 0x7fc08d611d80>, async_client=<openai.resources.embeddings.AsyncEmbeddings object at 0x7fc08dc41cc0>, model='text-embedding-ada-002', dimensions=None, deployment='text-embedding-ada-002', openai_api_version='2024-07-01-preview', openai_api_base=None, openai_api_type='azure', openai_proxy='', embedding_ctx_length=8191, openai_api_key=SecretStr('**********'), openai_organization=None, allowed_special=None, disallowed_special=None, chunk_size=2048, max_retries=2, request_timeout=None, headers=None, tiktoken_enabled=True, tiktoken_model_name=None, show_progress_bar=False, model_kwargs={}, skip_empty=False, default_headers=None, default_query=None, retry_min_seconds=4, retry_max_seconds=20, http_client=None, http_async_client=None, check_embedding_ctx_length=True, azure_endpoint='https://azureopenai-eastus-sr.openai.azure.com/', azure_ad_token=None, azure_ad_token_provider=None, validate_base_url=True)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "azure_openai_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6e8df8ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_store: AzureSearch = AzureSearch(\n",
    "    azure_search_endpoint=AZURE_AISEARCH_ENDPOINT,\n",
    "    azure_search_key=AZURE_AISEARCH_KEY,\n",
    "    index_name=azure_aisearch_index_name,\n",
    "    embedding_function=azure_openai_embeddings.embed_query,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f0986fcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding the documents into Azure AI Search index: index_06sep2024_104625\n",
      "\n",
      "Done\n",
      "Elapsed time = 0 minutes and 4 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "print(f\"Adding the documents into Azure AI Search index: {azure_aisearch_index_name}\")\n",
    "vector_store.add_documents(documents=chunks)\n",
    "\n",
    "elapsed = time.time() - start\n",
    "minutes, seconds = divmod(elapsed, 60)\n",
    "print(\"\\nDone\")\n",
    "print(f\"Elapsed time = {minutes:.0f} minutes and {seconds:.0f} seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "368252ae",
   "metadata": {},
   "source": [
    "### Index status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9d6884f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34m{\n",
      "     \"@odata.context\": \"https://azureaisearch-sr.search.windows.net/$metadata#indexes/$entity\",\n",
      "     \"@odata.etag\": \"\\\"0x8DCCE506E4AFCFB\\\"\",\n",
      "     \"name\": \"index_06sep2024_104625\",\n",
      "     \"defaultScoringProfile\": null,\n",
      "     \"fields\": [\n",
      "          {\n",
      "               \"name\": \"id\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": false,\n",
      "               \"filterable\": true,\n",
      "               \"retrievable\": true,\n",
      "               \"stored\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": true,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"dimensions\": null,\n",
      "               \"vectorSearchProfile\": null,\n",
      "               \"vectorEncoding\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"content\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": false,\n",
      "               \"retrievable\": true,\n",
      "               \"stored\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"dimensions\": null,\n",
      "               \"vectorSearchProfile\": null,\n",
      "               \"vectorEncoding\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"content_vector\",\n",
      "               \"type\": \"Collection(Edm.Single)\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": false,\n",
      "               \"retrievable\": true,\n",
      "               \"stored\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"dimensions\": 1536,\n",
      "               \"vectorSearchProfile\": \"myHnswProfile\",\n",
      "               \"vectorEncoding\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"metadata\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": false,\n",
      "               \"retrievable\": true,\n",
      "               \"stored\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"dimensions\": null,\n",
      "               \"vectorSearchProfile\": null,\n",
      "               \"vectorEncoding\": null,\n",
      "               \"synonymMaps\": []\n",
      "          }\n",
      "     ],\n",
      "     \"scoringProfiles\": [],\n",
      "     \"corsOptions\": null,\n",
      "     \"suggesters\": [],\n",
      "     \"analyzers\": [],\n",
      "     \"tokenizers\": [],\n",
      "     \"tokenFilters\": [],\n",
      "     \"charFilters\": [],\n",
      "     \"encryptionKey\": null,\n",
      "     \"similarity\": {\n",
      "          \"@odata.type\": \"#Microsoft.Azure.Search.BM25Similarity\",\n",
      "          \"k1\": null,\n",
      "          \"b\": null\n",
      "     },\n",
      "     \"semantic\": null,\n",
      "     \"vectorSearch\": {\n",
      "          \"algorithms\": [\n",
      "               {\n",
      "                    \"name\": \"default\",\n",
      "                    \"kind\": \"hnsw\",\n",
      "                    \"hnswParameters\": {\n",
      "                         \"metric\": \"cosine\",\n",
      "                         \"m\": 4,\n",
      "                         \"efConstruction\": 400,\n",
      "                         \"efSearch\": 500\n",
      "                    },\n",
      "                    \"exhaustiveKnnParameters\": null\n",
      "               },\n",
      "               {\n",
      "                    \"name\": \"default_exhaustive_knn\",\n",
      "                    \"kind\": \"exhaustiveKnn\",\n",
      "                    \"hnswParameters\": null,\n",
      "                    \"exhaustiveKnnParameters\": {\n",
      "                         \"metric\": \"cosine\"\n",
      "                    }\n",
      "               }\n",
      "          ],\n",
      "          \"profiles\": [\n",
      "               {\n",
      "                    \"name\": \"myHnswProfile\",\n",
      "                    \"algorithm\": \"default\",\n",
      "                    \"vectorizer\": null,\n",
      "                    \"compression\": null\n",
      "               },\n",
      "               {\n",
      "                    \"name\": \"myExhaustiveKnnProfile\",\n",
      "                    \"algorithm\": \"default_exhaustive_knn\",\n",
      "                    \"vectorizer\": null,\n",
      "                    \"compression\": null\n",
      "               }\n",
      "          ],\n",
      "          \"vectorizers\": [],\n",
      "          \"compressions\": []\n",
      "     }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "headers = {\"Content-Type\": \"application/json\", \"api-key\": AZURE_AISEARCH_KEY}\n",
    "params = {\"api-version\": \"2024-07-01\"}\n",
    "\n",
    "index_status = requests.get(\n",
    "    f\"{AZURE_AISEARCH_ENDPOINT}/indexes/{azure_aisearch_index_name}\", headers=headers, params=params\n",
    ")\n",
    "\n",
    "print(\"\\033[1;34m\", end=\"\")\n",
    "print(json.dumps(index_status.json(), indent=5))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2beb8a0",
   "metadata": {},
   "source": [
    "## 4. Azure OpenAI LLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ffe7605c",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = AzureChatOpenAI(\n",
    "    openai_api_version=AOAI_API_VERSION,\n",
    "    azure_deployment=AOAI_DEPLOYMENT_NAME,\n",
    "    azure_endpoint=AOAI_API_ENDPOINT,\n",
    "    openai_api_type=AOAI_API_TYPE,\n",
    "    temperature=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "03df054d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AzureChatOpenAI(client=<openai.resources.chat.completions.Completions object at 0x7fc08d7b8c10>, async_client=<openai.resources.chat.completions.AsyncCompletions object at 0x7fc08d7bbbe0>, temperature=0.0, openai_api_key=SecretStr('**********'), openai_proxy='', azure_endpoint='https://azureopenai-eastus-sr.openai.azure.com/', deployment_name='gpt-4o-mini', openai_api_version='2024-07-01-preview', openai_api_type='azure')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04ed8293",
   "metadata": {},
   "source": [
    "## 5. Q&A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8a322a1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = vector_store.as_retriever(\n",
    "    search_type=\"similarity\",\n",
    "    search_kwargs={\"k\": 3},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6ec4b46c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ChatPromptTemplate(input_variables=['context', 'question'], metadata={'lc_hub_owner': 'rlm', 'lc_hub_repo': 'rag-prompt', 'lc_hub_commit_hash': '50442af133e61576e74536c6556cefe1fac147cad032f4377b60c436e6cdcb6e'}, messages=[HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['context', 'question'], template=\"You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.\\nQuestion: {question} \\nContext: {context} \\nAnswer:\"))])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Getting a prompt template\n",
    "prompt = hub.pull(\"rlm/rag-prompt\")\n",
    "prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1b5c1b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_docs(docs):\n",
    "    \"\"\"\n",
    "    Format docs\n",
    "    \"\"\"\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4e21ff49",
   "metadata": {},
   "outputs": [],
   "source": [
    "rag_chain = ({\n",
    "    \"context\": retriever | format_docs,\n",
    "    \"question\": RunnablePassthrough()\n",
    "}\n",
    "             | prompt\n",
    "             | llm\n",
    "             | StrOutputParser())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "be910668",
   "metadata": {},
   "outputs": [],
   "source": [
    "def qna(question):\n",
    "    \"\"\"\n",
    "    QnA function to get a response for a given question.\n",
    "\n",
    "    This function takes a question as input and uses a response generation system\n",
    "    to produce an answer. It handles exceptions and displays the response in \n",
    "    colored output for better readability.\n",
    "\n",
    "    Parameters:\n",
    "    question (str): The question to be answered.\n",
    "    \"\"\"\n",
    "    print(\"\\033[1;34m\", end=\"\")\n",
    "\n",
    "    try:\n",
    "        response = rag_chain.invoke(question)\n",
    "        print(response)\n",
    "        print(\"\\033[1;32m\")\n",
    "        print(f\"Powered by Azure AI\")\n",
    "    except Exception as e:\n",
    "        print(\"\\033[1;31m\")\n",
    "        print(f\"[Error] {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4e3f618b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe language in the document is primarily English. It discusses the capabilities and performance of the phi-3 language model and its training data. Additionally, it mentions the model's limitations and future directions for multilingual capabilities.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What is the language in this document?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cf9fc53a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe companies mentioned in the document are Microsoft and UC Berkeley. Additionally, various models and baselines are referenced, including Claude 3 Haiku and GPT-4V-Turbo.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What are the companies mentionned in this document?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d5b17ca0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe technologies mentioned in the document include the Phi-3-Vision model, which is a multimodal model combining an image encoder (CLIP ViT-L/14) and a transformer decoder (phi-3-mini-128K-instruct). It utilizes a dynamic cropping strategy for high-resolution images and employs a pre-training phase with diverse datasets, including image-text pairs and synthetic data from OCR. Additionally, the model undergoes supervised finetuning and direct preference optimization for enhanced performance.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What are the technologies in this document?\"\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "04a8aa0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mPhi-3 refers to a series of language models developed by Microsoft, including phi-3-mini, phi-3-small, and phi-3-medium, with parameters ranging from 3.8 billion to 14 billion. These models are trained on extensive datasets and are designed for various applications, including chat and multimodal tasks. Phi-3-mini, in particular, is notable for its performance and safety alignment, making it suitable for deployment on devices like phones.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What is phi3?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "16198856",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe phi-3-mini model has 3.8 billion parameters and is designed for deployment on mobile devices, while the phi-3-small model has 7 billion parameters and offers improved performance. The phi-3-small model features a larger vocabulary size, a longer default context length, and utilizes advanced attention mechanisms for better efficiency. Overall, phi-3-small is significantly more capable than phi-3-mini in terms of language understanding and reasoning.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What are the differences between phi3 mini and phi3 small?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "47b5776b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe phi-3-mini model has 3.8 billion parameters and achieves 68.8% on the MMLU benchmark and 8.38 on MT-bench. In contrast, the phi-3-small model, with 7 billion parameters, performs better, scoring 75.7% on MMLU and 8.70 on MT-bench. Both models utilize a high-quality training dataset, but phi-3-small is designed for enhanced performance and efficiency.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What can you tell about the benchmarks for phi3 mini and phi3 small?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "415010c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mPhi3 Vision is a multimodal model with 4.2 billion parameters that processes images and textual prompts to generate textual outputs. It consists of an image encoder and a transformer decoder, utilizing a dynamic cropping strategy for high-resolution images. The model is pre-trained on a diverse dataset of image-text pairs and undergoes supervised fine-tuning and direct preference optimization for enhanced performance.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What is phi3 vision?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "55e7e9e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe weaknesses of these models include limited capacity for storing factual knowledge, which can lead to low performance on tasks like TriviaQA. Additionally, they may struggle with multilingual capabilities and are prone to issues such as factual inaccuracies, biases, and inappropriate content generation. While efforts are being made to mitigate these challenges, significant work remains to fully address them.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"How about the weakness of these models?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4f9b6287",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe comparison results on public and private multi-modal RAI benchmarks show that Phi-3-Vision outperforms other models, achieving higher scores in both internal and public benchmarks like RTVLM and VLGuard. Specifically, Phi-3-Vision scored 8.30 on the internal benchmark and 4.64 on RTVLM, while its performance on VLGuard was 9.12. The results indicate that safety post-training significantly enhances RAI performance across nearly all categories.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"How about the comparison results on public and private multi-modal RAI benchmarks?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad147157",
   "metadata": {},
   "source": [
    "### With references"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "980dd1d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mThe vision model, Phi-3-Vision, has 4.2 billion parameters. It is designed to process both images and textual prompts. This model combines an image encoder and a transformer decoder for multimodal tasks.\n",
      "\u001b[1;32m\n",
      "Powered by Azure AI\n"
     ]
    }
   ],
   "source": [
    "question = \"What is the number of parameters of the vision model?\"\n",
    "\n",
    "qna(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "0482b405",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34m"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'documents': [{'Header 1': '6 Phi-3-Vision',\n",
       "   'Header 2': '6.1 Technical Specifications'},\n",
       "  {'Header 1': '6 Phi-3-Vision', 'Header 2': '6.2 Academic benchmarks'},\n",
       "  {'Header 1': 'Abstract'},\n",
       "  {'Header 1': 'Introduction 1'}],\n",
       " 'answer': 'The vision model, Phi-3-Vision, has 4.2 billion parameters. It is designed to process both images and textual prompts. This model combines an image encoder and a transformer decoder for multimodal tasks.'}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rag_chain_from_docs = (\n",
    "    {\n",
    "        \"context\": lambda input: format_docs(input[\"documents\"]),\n",
    "        \"question\": itemgetter(\"question\"),\n",
    "    }\n",
    "    | prompt\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    ")\n",
    "\n",
    "rag_chain_with_source = RunnableMap(\n",
    "    {\"documents\": retriever, \"question\": RunnablePassthrough()}\n",
    ") | {\n",
    "    \"documents\": lambda input: [doc.metadata for doc in input[\"documents\"]],\n",
    "    \"answer\": rag_chain_from_docs,\n",
    "}\n",
    "\n",
    "print(\"\\033[1;34m\", end=\"\")\n",
    "rag_chain_with_source.invoke(question)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e1bf3bb",
   "metadata": {},
   "source": [
    "## 6. Webapp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "282fb51d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def webapp_function(prompt):\n",
    "    \"\"\"\n",
    "    Webapp function\n",
    "    \"\"\"\n",
    "    return rag_chain.invoke(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "43091a53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7862\n",
      "Running on public URL: https://b40c74b8af7536d1b4.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://b40c74b8af7536d1b4.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions_list = [\n",
    "    \"What is the language in this document?\",\n",
    "    \"Classify this document into one of these categories: ART, MUSIC, AI, POLITICS\",\n",
    "    \"What are the technologies in this document?\",\n",
    "    \"What is phi3?\",\n",
    "    \"What are the differences between phi3 mini and phi3 small?\",\n",
    "    \"What can you tell about the benchmarks for phi3 mini and phi3 small?\",\n",
    "    \"How about the comparison results on public and private multi-modal RAI benchmarks?\",\n",
    "]\n",
    "\n",
    "footnote = \"Powered by Azure AI services\"\n",
    "logo = \"https://github.com/retkowsky/images/blob/master/adi_logo.jpg?raw=true\"\n",
    "image = \"<center> <img src= {} width=150px></center>\".format(logo)\n",
    "title = \"RAG with Azure Document Intelligence and Azure OpenAI GPT-4o mini\"\n",
    "theme = \"snehilsanyal/scikit-learn\"  # Themes: https://huggingface.co/spaces/gradio/theme-gallery\n",
    "\n",
    "webapp = gr.Interface(\n",
    "    fn=webapp_function,\n",
    "    inputs=gr.Textbox(label=\"Enter your prompt:\"),\n",
    "    outputs=gr.Textbox(label=\"Azure AI answer:\"),\n",
    "    title=title,\n",
    "    description=image,\n",
    "    article=footnote,\n",
    "    examples=questions_list,\n",
    "    theme=theme,\n",
    ")\n",
    "\n",
    "\n",
    "# Launch the web app\n",
    "webapp.launch(share=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3332db8d",
   "metadata": {},
   "source": [
    "## 7. Deleting the index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "084305ca",
   "metadata": {},
   "source": [
    "### Index status & size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "712f5f89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Azure Cognitive Search index status for: index_06sep2024_104625\n",
      "\n",
      "\u001b[1;34m{\n",
      "  \"@odata.context\": \"https://azureaisearch-sr.search.windows.net/$metadata#Microsoft.Azure.Search.V2024_07_01.IndexStatistics\",\n",
      "  \"documentCount\": 17,\n",
      "  \"storageSize\": 609003,\n",
      "  \"vectorIndexSize\": 106464\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Construct the request URL\n",
    "url = f\"{AZURE_AISEARCH_ENDPOINT}/indexes/{azure_aisearch_index_name}/stats?api-version=2024-07-01\"\n",
    "\n",
    "# Set the request headers\n",
    "headers = {\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"api-key\": AZURE_AISEARCH_KEY,\n",
    "}\n",
    "\n",
    "# Make the GET request\n",
    "response = requests.get(url, headers=headers)\n",
    "print(f\"Azure Cognitive Search index status for: {azure_aisearch_index_name}\\n\")\n",
    "\n",
    "# Check the response status code\n",
    "if response.status_code == 200:\n",
    "    print(\"\\033[1;34m\", end=\"\")\n",
    "    data = response.json()\n",
    "    print(json.dumps(data, indent=2))\n",
    "\n",
    "else:\n",
    "    print(f\"Request failed with status code: {response.status_code}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "6bc8cab1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;34mNumber of documents = 17\n",
      "Total storage of the index = 594.73 Kb\n"
     ]
    }
   ],
   "source": [
    "print(\"\\033[1;34m\", end=\"\")\n",
    "document_count = data['documentCount']\n",
    "storage_size = data['storageSize'] / 1024\n",
    "print(f\"Number of documents = {document_count}\\nTotal storage of the index = {storage_size:.2f} Kb\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59d69459",
   "metadata": {},
   "source": [
    "### Post Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "bbf8280d",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_client = SearchIndexClient(\n",
    "    endpoint=AZURE_AISEARCH_ENDPOINT,\n",
    "    credential=AzureKeyCredential(AZURE_AISEARCH_KEY),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "62be51f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deleting the index\n",
    "search_client.delete_index(azure_aisearch_index_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e5e9df0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10 - SDK v2",
   "language": "python",
   "name": "python310-sdkv2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}