|
|
import os |
|
|
import joblib |
|
|
import langchain |
|
|
import streamlit as st |
|
|
import pickle as pkl |
|
|
from langchain.chains import RetrievalQAWithSourcesChain |
|
|
from langchain_community.document_loaders import UnstructuredURLLoader |
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter |
|
|
from langchain_community.embeddings import SentenceTransformerEmbeddings |
|
|
from langchain_community.vectorstores import Chroma, FAISS |
|
|
from langchain_openai import ChatOpenAI |
|
|
from dotenv import load_dotenv |
|
|
import time |
|
|
|
|
|
|
|
|
load_dotenv("ping.env") |
|
|
api_key=os.getenv("OPENAI_API_KEY") |
|
|
api_base=os.getenv("OPENAI_API_BASE") |
|
|
|
|
|
llm=ChatOpenAI(model_name="google/gemma-3n-e2b-it:free",temperature=0) |
|
|
try: |
|
|
with open("embedmo.pkl", "rb") as f: |
|
|
m1 = pkl.load(f) |
|
|
|
|
|
if not isinstance(m1, SentenceTransformerEmbeddings): |
|
|
raise ValueError("Loaded object is not a SentenceTransformerEmbeddings instance.") |
|
|
except Exception as e: |
|
|
st.error(f"Failed to load embedding model: {str(e)}") |
|
|
st.stop() |
|
|
|
|
|
m2=joblib.load("m1.joblib") |
|
|
st.title("URL ANALYSER🔗") |
|
|
st.sidebar.title("Give your URls🔗?") |
|
|
mp=st.empty() |
|
|
|
|
|
|
|
|
url1=st.sidebar.text_input(f"URL 1🔗") |
|
|
url2=st.sidebar.text_input(f"URL 2🔗") |
|
|
url3=st.sidebar.text_input(f"URL 3🔗") |
|
|
|
|
|
|
|
|
purs=st.button("gotcha") |
|
|
if purs: |
|
|
|
|
|
st.write(url1) |
|
|
st.write(url2) |
|
|
st.write(url3) |
|
|
mp.text("Loading..URl..Loader....☑️☑️☑️") |
|
|
sic=UnstructuredURLLoader(urls=[url1,url2,url3]) |
|
|
docs=sic.load() |
|
|
st.write(len(docs)) |
|
|
mp.text("Loading..txt..splitter....☑️☑️☑️") |
|
|
tot=RecursiveCharacterTextSplitter.from_tiktoken_encoder(encoding_name="cl100k_base",chunk_size=512,chunk_overlap=16) |
|
|
doccs=tot.split_documents(docs) |
|
|
st.write(len(doccs)) |
|
|
mp.text("Loading..VB...☑️☑️☑️") |
|
|
vv=Chroma.from_documents(doccs,m1) |
|
|
r2=vv.as_retriever(search_type="similarity",search_kwargs={"k":4}) |
|
|
mp.text("Loading..Retri....☑️☑️☑️") |
|
|
ra1=RetrievalQAWithSourcesChain.from_chain_type(llm=llm,retriever=r2,chain_type="map_reduce") |
|
|
st.session_state.ra1=ra1 |
|
|
mp.text("DB & Retri Done ✅✅✅") |
|
|
time.sleep(3) |
|
|
query=mp.text_input("UR Question??") |
|
|
if query: |
|
|
if "ra1" not in st.session_state: |
|
|
st.warning("pls give ur urls") |
|
|
else: |
|
|
with st.spinner("Wait for it..."): |
|
|
result=st.session_state.ra1({"question":query},return_only_outputs=True) |
|
|
st.header("Answer") |
|
|
st.subheader(result["answer"]) |
|
|
g = st.button("Source") |
|
|
if g: |
|
|
sources = result.get("sources", "") |
|
|
st.subheader("Sources") |
|
|
for line in sources.split("\n"): |
|
|
st.write(line) |
|
|
|
|
|
|
|
|
|
