WARecordRDD

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def discardContent(contentREs: Set[Regex]): RDD[ArchiveRecord]

   

7. def discardDate(date: String): RDD[ArchiveRecord]

   

8. def discardDomains(urls: Set[String]): RDD[ArchiveRecord]

   

9. def discardMimeTypes(mimeTypes: Set[String]): RDD[ArchiveRecord]

   

   Filters MimeTypes from RDDs.

10. def discardUrlPatterns(urlREs: Set[Regex]): RDD[ArchiveRecord]

    

11. def discardUrls(urls: Set[String]): RDD[ArchiveRecord]

    

12. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

13. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

14. def extractHyperlinksDF(): DataFrame

    

15. def extractImageDetailsDF(): DataFrame

    

16. def extractImageLinksDF(): DataFrame

    

17. def extractValidPagesDF(): DataFrame

    

18. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

19. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

20. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

21. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

22. def keepContent(contentREs: Set[Regex]): RDD[ArchiveRecord]

    

    Removes all content that does not pass Regular Expression test.

    Removes all content that does not pass Regular Expression test.

    contentREs

    a list of Regular expressions to keep

23. def keepDate(dates: List[String], component: DateComponent = DateComponent.YYYYMMDD): RDD[ArchiveRecord]

    

    Removes all data that does not have selected data.

    Removes all data that does not have selected data.

    dates

    a list of dates to keep

    component

    the selected DateComponent enum value

24. def keepDomains(urls: Set[String]): RDD[ArchiveRecord]

    

    Removes all data but selected source domains.

    Removes all data but selected source domains.

    urls

    a Set of urls for the source domains to keep

25. def keepImages(): RDD[ArchiveRecord]

    

    Removes all data except images.

26. def keepLanguages(lang: Set[String]): RDD[ArchiveRecord]

    

    Removes all data not in selected language.

    Removes all data not in selected language.

    lang

    a Set of ISO 639-2 codes

27. def keepMimeTypes(mimeTypes: Set[String]): RDD[ArchiveRecord]

    

    Removes all data but selected mimeTypes.

    Removes all data but selected mimeTypes.

    mimeTypes

    a Set of Mimetypes to keep

28. def keepUrlPatterns(urlREs: Set[Regex]): RDD[ArchiveRecord]

    

    Removes all data but selected url patterns.

    Removes all data but selected url patterns.

    urlREs

    a Set of Regular Expressions to keep

29. def keepUrls(urls: Set[String]): RDD[ArchiveRecord]

    

    Removes all data but selected exact URLs

    Removes all data but selected exact URLs

    urls

    a Set of URLs to keep

30. def keepValidPages(): RDD[ArchiveRecord]

    

    Removes all non-html-based data (images, executables etc.) from html text.

31. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

32. final def notify(): Unit

    

    Definition Classes

    AnyRef

33. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

34. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

35. def toString(): String

    

    Definition Classes

    AnyRef → Any

36. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

37. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

38. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )