using OpenQA.Selenium;
using System;
using System.Threading;

namespace MyNameSpace {
  public static partial class Extensions {
    #region --- safe click ----------------------------------------------------------------
    public static void SafeClick(this IWebElement element, int intervalInMilliseconds = 25, int timeoutInMilliseconds = 200) {
      bool success = false;
      int  counter = 0;
      while (!success && counter < timeoutInMilliseconds) {
        try {
          Thread.Sleep(TimeSpan.FromMilliseconds(intervalInMilliseconds));
          element.Click();
          success = true;
          return;
        } catch (Exception ex) {
          counter += intervalInMilliseconds;
        }
      }
    }
    #endregion
  }
}}

using HtmlAgilityPack;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using System;
using System.Collections.Generic;

namespace MyNameSpace {
  public static class IMDbScraper {
    public static HtmlDocument ScrapeIMDbPage(string imdbID) {
      // --- create Selenium service and driver ----------------------------------------------------------
      ChromeDriverService driverService = ChromeDriverService.CreateDefaultService();
      driverService.HideCommandPromptWindow = true;

      ChromeOptions chromeOptions = new ChromeOptions();
      chromeOptions.AddArguments(
        "--blink-settings=imagesEnabled=false"
      );
      chromeOptions.AddUserProfilePreference("profile.managed_default_content_settings.images", 2);
      chromeOptions.AddUserProfilePreference("profile.default_content_setting_values.images", 2);
      IWebDriver driver = new ChromeDriver(driverService, chromeOptions);

      // --- call url in Selenium browser ----------------------------------------------------------------
      string url = String.Format("https://www.imdb.com/title/{0}/companycredits/", imdbID); // e.g. sub page companycredits
      driver.Navigate().GoToUrl(url);

      // --- find and click on any "more" buttons --------------------------------------------------------
      IReadOnlyCollection<IWebElement> elements = driver.FindElements(By.ClassName("ipc-see-more__text"));

      if (elements != null) {
        IJavaScriptExecutor javaScript = (IJavaScriptExecutor)driver;
        bool doIt = true;
        foreach (WebElement element in elements) {
          if (doIt) {
            try {
              if (element.Location.Y > 100) {
                string script = String.Format("window.scrollTo({0}, {1})", 0, element.Location.Y - 200);
                javaScript.ExecuteScript(script); // execute JavaScript to scroll to the button
              }

              element.SafeClick(); // element.Click() is buggy and crashes, therefore we use our extension method
              doIt = false;
            } catch { }
          } else {
            doIt = true;
          }
        }
      }

      // --- get body as HtmlDocument for further HtmlAgilityPack parsing --------------------------------
      HtmlDocument result = new HtmlDocument();
      result.LoadHtml(driver.FindElement(By.XPath(@"//body")).GetAttribute("innerHTML"));

      return result;
    }

    public static List<Company> ParseProductionCompanies(HtmlDocument document) {
      List<Company> result = new List<Company>(); // own class: see below

      // --- get node of desired section -----------------------------------------------------------------
      string path = @"//div[@data-testid=""sub-section-production""]";
      HtmlNode node = document.DocumentNode.SelectSingleNode(path);

      // --- parse content -------------------------------------------------------------------------------
      if (node != null) {
        try {
          foreach (HtmlNode entry in node.ChildNodes[0].ChildNodes) {
            if (entry.Name != "li") {
              continue;
            }

            Company company = new Company() {
              Sphere = Sphere.Production // own Enum: see below
            };

            try {
              company.ID = entry.ChildNodes[0]
                                .Attributes["href"]
                                .Value
                                .GetSubstringBetweenStrings("/company/", "?ref"); // own extension method for string: create it yourself ;)

              company.Name = entry.ChildNodes[0].InnerText;
            } catch { }

            try {
              HtmlNode details = entry.ChildNodes[1]
                                      .ChildNodes[0]
                                      .ChildNodes[0];

              try {
                company.Remark = details.ChildNodes[0].InnerText;
              } catch { }
            } catch { }

            result.Add(company);
          }
        } catch { }
      }

      return result;
    }
  }

  public class Company {
    public string Country { get; set; }
    public string ID      { get; set; }
    public string Name    { get; set; }
    public string Remark  { get; set; }
    public Sphere Sphere  { get; set; }
    public int    Year    { get; set; }
  }

  public enum Sphere {
    [Description("Distributor")]
    Distributor,
    
    [Description("Other")]
    Other,

    [Description("Production")]
    Production,
    
    [Description("Special Effects")]
    SpecialEffects
  };
}