Spaces:

WebShepherd
/

Project-Web-Shepherd

Running

File size: 16,281 Bytes

<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <meta name="description"
        content="Web-Shepherd: Advancing PRMs for Reinforcing Web Agents">
  <meta name="keywords" content="Nerfies, D-NeRF, NeRF">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Web-Shepherd: Advancing PRMs for Reinforcing Web Agents</title>

  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
        rel="stylesheet">

  <link rel="stylesheet" href="./static/css/bulma.min.css">
  <link rel="stylesheet" href="./static/css/bulma-carousel.min.css">
  <link rel="stylesheet" href="./static/css/bulma-slider.min.css">
  <link rel="stylesheet" href="./static/css/fontawesome.all.min.css">
  <link rel="stylesheet"
        href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
  <link rel="stylesheet" href="./static/css/index.css">
  <link rel="icon" href="./static/images/favicon.svg">

  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
  <script defer src="./static/js/fontawesome.all.min.js"></script>
  <script src="./static/js/bulma-carousel.min.js"></script>
  <script src="./static/js/bulma-slider.min.js"></script>
  <script src="./static/js/index.js"></script>
</head>
<body>

<section class="hero">
  <div class="hero-body">
    <div class="container is-max-desktop">
      <div class="columns is-centered">
        <div class="column has-text-centered">
          <h1 class="title is-1 publication-title">
            <img src="static/images/shepherd_emoji.png" style="width:1em;vertical-align: middle" alt="Logo"/> 
            Web-Shepherd:
          </h1>
          <h2 class="subtitle is-3 publication-subtitle">
            Advancing PRMs for Reinforcing Web Agents
          </h2>
          <div class="is-size-5 publication-authors">
            <span class="author-block">
              <span target="_blank">Anonymous Authors</span></span>
          </div>

          <div class="is-size-10 publication-authors", style="margin-top: 1em;">
            <span class="author-block">Note that this project page is fully anonymized. Some links might not be available due to anonymization.</span>
          </div>

          <div class="column has-text-centered">
            <div class="publication-links">
              <!-- PDF Link. -->
              <!-- <span class="link-block">
                <a href="https://arxiv.org/pdf/2011.12948" target="_blank"
                   class="external-link button is-normal is-rounded is-dark">
                  <span class="icon">
                      <i class="fas fa-file-pdf"></i>
                  </span>
                  <span>Paper</span>
                </a>
              </span> -->
              <!-- <span class="link-block">
                <a href="https://arxiv.org/abs/2011.12948" target="_blank"
                   class="external-link button is-normal is-rounded is-dark">
                  <span class="icon">
                      <i class="ai ai-arxiv"></i>
                  </span>
                  <span>arXiv</span>
                </a>
              </span> -->
              <!-- Video Link. -->
              <!-- <span class="link-block">
                <a href="https://www.youtube.com/watch?v=MrKrnHhk8IA" target="_blank"
                   class="external-link button is-normal is-rounded is-dark">
                  <span class="icon">
                      <i class="fab fa-youtube"></i>
                  </span>
                  <span>Video</span>
                </a>
              </span> -->
              <!-- Code Link. -->
              <span class="link-block">
                <a href="https://huggingface.co/WebShepherd/WebShepherd_8B"
                   class="external-link button is-normal is-rounded is-dark" target="_blank">
                  <span class="icon">
                    <p style="font-size:18px">🤗</p>
                  </span>
                  <span>Models</span>
                </a>
              </span>
              <!-- Dataset Link. -->
              <span class="link-block">
                <a href="https://huggingface.co/datasets/WebShepherd/WebPRMCollection_preference_pair"
                   class="external-link button is-normal is-rounded is-dark" target="_blank">
                  <span class="icon">
                    <p style="font-size:18px">🤗</p>
                  </span>
                  <span>Datasets</span>
                </a>
              </span>
              <span class="link-block">
                <a href="https://huggingface.co/datasets/WebShepherd/WebRewardBench"
                   class="external-link button is-normal is-rounded is-dark" target="_blank">
                  <span class="icon">
                    <p style="font-size:18px">🤗</p>
                  </span>
                  <span>Benchmark</span>
                </a>
              </span>
            </div>
            <div class="links-row">
              <span class="link-block">
                <a href="#mainresults"
                   class="external-link button is-normal is-rounded is-dark">
                  <span class="icon has-text-white">
                    <i class="fa-solid fa-trophy"></i>
                      <!-- <p style="font-size:18px">🏆</p> -->
                  </span>
                  <span>Main Results</span>
                </a>
              </span>
          </div>
        </div>
      </div>
    </div>
  </div>
</section>

</section>
<style>
  .center {
    display: block;
    margin-left: auto;
    margin-right: auto;
    width: 80%;
  }
  </style>
<section class="hero teaser">
  <div class="container is-max-desktop">
        <div class="content has-text-centered">
          <img src="static/images/figure_1.png" alt="geometric reasoning" width="95%"/>
          <p> Performance and cost-efficiency of Web-Shepherd (3B). Web-Shepherd achieves the state-of-the-art performance while requiring significantly lower cost compared to existing baselines. </p>
        </div>
      <!-- </div> -->
    </div>
  </div>
</section>

<section class="section">
  <div class="container is-max-desktop">
    <!-- Abstract. -->
    <div class="columns is-centered has-text-centered">
      <div class="column is-four-fifths">
        <h2 class="title is-3">Introduction</h2>
        <div class="content has-text-justified">
          <p>
            Web navigation is a unique domain that can automate many repetitive real-life tasks and is challenging as it requires long-horizon sequential decision making beyond typical multimodal large language model (MLLM) tasks.
          </p>
          <p>
            Yet, specialized reward models for web navigation that can be utilized during both training and test-time have been absent until now. Despite the importance of speed and cost-effectiveness, prior works have utilized MLLMs as reward models, which poses significant constraints for real-world deployment.
            To address this, in this work, we propose the first process reward model (PRM) called Web-Shepherd which could assess web navigation trajectories in a step-level. To achieve this, we first construct the WEBPRM collection, a large-scale dataset with 40K step-level preference pairs and annotated checklists spanning diverse domains and difficulty levels. Next, we also introduce the WEB-RewardBench, the first meta-evaluation benchmark for evaluating PRMs. In our experiments, we observe that our Web-Shepherd achieves about 30 points better accuracy compared to using GPT-4o on WEB-RewardBench.
          </p>
          <p>
            Furthermore, when testing on WebArena-lite by using GPT-4o-mini as the policy and Web-Shepherd as the verifier, we achieve 10.3 points better performance, in 10 times less cost compared to using GPT-4o-mini as the verifier.
          </p>
        </div>
      </div>
    </div>
    <!--/ Abstract. -->
  </div>
</section>

<section class="hero is-light is-small">
  <div class="hero-body has-text-centered">
  <h1 class="title is-1 mmmu">
    <span class="mmmu" style="vertical-align: middle">WEBPRM Collection</span>
  </h1>
  </div>
</section>

<section class="section">
  <div class="container">
    <div class="columns is-centered has-text-centered">
      <!-- <div class="column is-full-width has-text-centered"> -->
      <div class="column is-four-fifths">
        <h2 class="title is-3">Overview</h2>
        <div class="content has-text-centered">
          <img src="static/images/WPRMCollection.svg" alt="algebraic reasoning" class="center" style="width:100%">
          <p> An overview of the dataset collection process of WEBPRM</p>
        </div>
        <div class="content has-text-justified">
          <p>
            Building Preference Reward Models (PRMs) for web agents presents a core challenge: the lack of a high-quality, task-aligned dataset. To address this, we introduce WEBPRM, the dataset explicitly designed for training PRMs in the context of web-based agents.
          </p>
          <p>
            We collect expert demonstrations from trained annotators across websites accessible via Playwright, based on the Mind2Web benchmark. All annotators undergo a three-hour training to ensure high-quality and consistent behavior modeling. Each interaction is reviewed by a panel of human evaluators, and we filter out ambiguous or irreproducible samples.
          </p>
        </div>
    </div>
    </div>
    </div>
  </div>
</section>

<section class="hero is-light is-small">
  <div class="hero-body has-text-centered">
  <h1 class="title is-1 mmmu">
    <span class="mmmu" style="vertical-align: middle">Web-Shepherd</span>
  </h1>
  </div>
</section>

<section class="section">
  <div class="container">
    <div class="columns is-centered has-text-centered">
      <!-- <div class="column is-full-width has-text-centered"> -->
      <div class="column is-four-fifths">
        <h2 class="title is-3">Overview</h2>
        <div class="content has-text-centered">
          <img src="static/images/WebShepherd.svg" alt="algebraic reasoning" class="center" style="width:100%">
          <p> An overview of the Web-Shepherd</p>
        </div>
        <div class="content has-text-justified">
          <p>
            We introduce Web-Shepherd, a process reward model designed to provide dense and reliable supervision to web agents and enable more informative credit assignment.
          </p>
          <p>
            We train Web-Shepherd on the WEBPRM Collection to support two key functionalities: (1) generating task-specific checklists, and (2) assigning rewards based on checklist completion.
          </p>
        </div>
    </div>
    </div>
    </div>
  </div>
</section>
</section>
<!-- RESULTS SECTION -->
<section class="hero is-light is-small">
  <div class="hero-body has-text-centered">
    <h1 class="title is-1 mmmu" id="mainresults">Main Results</h1>
  </div>
</section>

<section class="section">
  <div class="container">
<!-------------------------------------------------------------------- RESULTS SECTION -------------------------------------------------------------------->
    <div class="columns is-centered m-6">
        <div class="column is-full has-text-centered content">
          <!-- <h2 class="title is-3" id="leaderboard"></h2> -->
          <div class="content">

          <div class="content has-text-centered">
            <img src="static/images/main_result.png" alt="algebraic reasoning"  width="100%"/>
            <p>Evaluation results on WEB-RewardBench. T: text observation, I: image observation</p>
          </div>

            <div class="content has-text-justified">
              <p>
                Table above reports the evaluation results on WEB-RewardBench. As shown in Table, state-of-the-art MLLMs struggle to provide reliable rewards for web navigation tasks. This limitation is particularly evident in the trajectory accuracy metric. In this measure, models frequently fail to assign correct rewards consistently at each time step within a single task. In contrast, Web-Shepherd significantly outperforms all baselines, demonstrating a substantial performance gap across all benchmark settings.
              </p>
              <p>
                Also, Table above demonstrates that both baseline and our models benefit significantly from the checklist in assigning rewards.
                Checklists lead to more accurate and consistent reward assignments, as evidenced by improvements in trajectory accuracy across all baselines.
                These results suggests that checklists serve as valuable guidance, helping models maintain coherence in predicting the process reward.
              </p>
            </div>

          </div>
        </div>
    </div>
  </div>
</section>


<!-- <section class="section">
  <div class="container is-max-desktop">
    <div class="columns is-centered">
      <div class="column is-full-width">
        <h2 class="title is-3">Related Links</h2>

        <div class="content has-text-justified">
          <p>
            There's a lot of excellent work that was introduced around the same time as ours.
          </p>
          <p>
            <a href="https://arxiv.org/abs/2104.09125" target="_blank">Progressive Encoding for Neural Optimization</a> introduces an idea similar to our windowed position encoding for coarse-to-fine optimization.
          </p>
          <p>
            <a href="https://www.albertpumarola.com/research/D-NeRF/index.html" target="_blank">D-NeRF</a> and <a href="https://gvv.mpi-inf.mpg.de/projects/nonrigid_nerf/" target="_blank">NR-NeRF</a>
            both use deformation fields to model non-rigid scenes.
          </p>
          <p>
            Some works model videos with a NeRF by directly modulating the density, such as <a href="https://video-nerf.github.io/" target="_blank">Video-NeRF</a>, <a href="https://www.cs.cornell.edu/~zl548/NSFF/" target="_blank">NSFF</a>, and <a href="https://neural-3d-video.github.io/" target="_blank">DyNeRF</a>
          </p>
          <p>
            There are probably many more by the time you are reading this. Check out <a href="https://dellaert.github.io/NeRF/" target="_blank">Frank Dellart's survey on recent NeRF papers</a>, and <a href="https://github.com/yenchenlin/awesome-NeRF" target="_blank">Yen-Chen Lin's curated list of NeRF papers</a>.
          </p>
        </div>
      </div>
    </div>

  </div>
</section> -->


<!-- <section class="section" id="BibTeX">
  <div class="container is-max-desktop content">
    <h2 class="title">BibTeX</h2>
    <pre><code>@article{park2021nerfies,
  author    = {Park, Keunhong and Sinha, Utkarsh and Barron, Jonathan T. and Bouaziz, Sofien and Goldman, Dan B and Seitz, Steven M. and Martin-Brualla, Ricardo},
  title     = {Nerfies: Deformable Neural Radiance Fields},
  journal   = {ICCV},
  year      = {2021},
}</code></pre>
  </div>
</section> -->


<footer class="footer">
  <div class="container">
    <div class="content has-text-centered">
      <a class="icon-link" target="_blank"
         href="./static/videos/nerfies_paper.pdf">
        <i class="fas fa-file-pdf"></i>
      </a>
      <a class="icon-link" href="https://github.com/keunhong" target="_blank" class="external-link" disabled>
        <i class="fab fa-github"></i>
      </a>
    </div>
    <div class="columns is-centered">
      <div class="column is-8">
        <div class="content">
          <p>
            This website is licensed under a <a rel="license" target="_blank"
                                                href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
            Commons Attribution-ShareAlike 4.0 International License</a>.
          </p>
          <p>
            This means you are free to borrow the <a target="_blank"
              href="https://github.com/nerfies/nerfies.github.io">source code</a> of this website,
            we just ask that you link back to this page in the footer.
            Please remember to remove the analytics code included in the header of the website which
            you do not want on your website.
          </p>
        </div>
      </div>
    </div>
  </div>
</footer>

</body>
</html>