最新ニュースをChatGPTで問い合わせ

import logging
import os
import environ
import chromadb
import feedparser
from datetime import datetime, timedelta, date
from django.conf import settings
from django.core.management.base import BaseCommand
from langchain.vectorstores import Chroma
from langchain.chat_models import ChatOpenAI
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.chains import ConversationalRetrievalChain
from chromadb.config import Settings
from chromadb.utils import embedding_functions
from newspaper import Article, Config

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument('attrs', nargs='+', type=str)
                            
    def getcontent(self, days):
        yesterday = datetime.now() - timedelta(days)
        
        client = chromadb.Client(Settings(
            chroma_db_impl="duckdb+parquet",
            persist_directory=os.environ["CHROMA_DB_DIR"]
        ))
        openai_ef = embedding_functions.OpenAIEmbeddingFunction(
            api_key=os.environ["OPENAI_API_KEY"],
            model_name="text-embedding-ada-002"
        )
        collection = client.get_or_create_collection(
                               name="langchain" + yesterday.strftime('%Y%m%d'),
                               embedding_function=openai_ef)

        # fetch feed
        rss = feedparser.parse('https://www.gmo.jp/rss/')
        for item in rss.entries:
            updateTime=  datetime.strptime(item.published,
                                          '%Y-%m-%d %H:%M:%S').strftime('%Y/%m/%d')
            if updateTime == yesterday.strftime('%Y/%m/%d'):
                try:
                    config = Config()
                    config.request_timeout = 20
                    website = Article(item.link, language='ja', config=config)
                    website.download()
                    website.parse()
                    website.nlp()
                    
                    text_splitter = CharacterTextSplitter(        
                        separator = "\n\n",
                        chunk_size = 400,
                        chunk_overlap  = 0,
                        length_function = len,
                    )
                    docs = text_splitter.create_documents([website.text],
                                                          metadatas=[{"source": item.link}])
                    for idx, doc in enumerate(docs):
                        collection.add(documents=[doc.page_content,],
                                       metadatas=[doc.metadata,],
                                       ids=[doc.metadata['source']+"_"+str(idx),])

                except Exception as e:
                    logger.error(e)
                    continue

        client.persist()


    def select_day_article(self, days):
        yesterday = date.today() - timedelta(days)
        print (f"{yesterday.strftime('%Y/%m/%d')}の記事について")

        embeddings = OpenAIEmbeddings()
        vectordb = Chroma(persist_directory=os.environ["CHROMA_DB_DIR"],
　　　　　　　　　　　　　　collection_name="langchain" + yesterday.strftime('%Y%m%d'),
　　　　　　　　　　　　　　embedding_function=embeddings)
        retriever = vectordb.as_retriever()
        retriever.search_kwargs['distance_metric'] = 'cos'
        retriever.search_kwargs['fetch_k'] = 40
        retriever.search_kwargs['maximal_marginal_relevance'] = True
        retriever.search_kwargs['k'] = 8
        llm = ChatOpenAI(temperature=0.7, model_name="gpt-3.5-turbo")
        # llm = ChatOpenAI(temperature=0.7, model_name="gpt-4")
        qa = ConversationalRetrievalChain.from_llm(llm, retriever, return_source_documents=True)

        questions = ["GMOについての記事をざっくり要点をまとめて日本語で箇条書きにして教えて下さい。",
                     "その記事の内容で注目する重要なポイントを箇条書きにして３点、日本語で教えて下さい。",
                     "そのポイントは今後はどういう将来性・ビジネスになっていくのか？日本語で教えて下さい。"]
        chat_history = []
        for q in questions:
            result = qa({"question": q, "chat_history": chat_history})
            chat_history.append((q, result['answer']))
            print(f"**質問**: \n{q} \n")
            print(f"**回答**: \n{result['answer']} \n")

            
    def handle(self, *args, **options):
        env = environ.Env()
        env.read_env(os.path.join(settings.BASE_DIR, ".env"))
        os.environ["OPENAI_API_KEY"] = os.environ["OPENAI_API_KEY"]

        attrs = options['attrs']
        if attrs[0] == "crawl":            
            self.getcontent(int(attrs[1]))
        elif attrs[0] == "article":
            self.select_day_article(int(attrs[1]))