reviews.conf 2.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172
  1. # Specify server locations in a SOLR_LOCATOR variable; used later in
  2. # variable substitutions:
  3. SOLR_LOCATOR : {
  4. # Name of solr collection
  5. collection : reviews
  6. # ZooKeeper ensemble
  7. zkHost : "127.0.0.1:2181/solr"
  8. }
  9. # Specify an array of one or more morphlines, each of which defines an ETL
  10. # transformation chain. A morphline consists of one or more (potentially
  11. # nested) commands. A morphline is a way to consume records (e.g. Flume events,
  12. # HDFS files or blocks), turn them into a stream of records, and pipe the stream
  13. # of records through a set of easily configurable transformations on the way to
  14. # a target application such as Solr.
  15. morphlines : [
  16. {
  17. id : reviews
  18. importCommands : ["com.cloudera.**"]
  19. commands : [
  20. {
  21. readCSV {
  22. separator : ","
  23. columns : [business_id, cool, date, funny, id, stars, text, type, useful, user_id, name, city, full_address, latitude, longitude, neighborhoods, open, review_count, state]
  24. quoteChar : "\""
  25. }
  26. }
  27. {
  28. if {
  29. conditions : [
  30. {
  31. equals { id : [] }
  32. }
  33. ]
  34. then : [
  35. {
  36. dropRecord {}
  37. }
  38. ]
  39. }
  40. }
  41. # Consume the output record of the previous command and pipe another
  42. # record downstream.
  43. #
  44. # Command that deletes record fields that are unknown to Solr
  45. # schema.xml.
  46. #
  47. # Recall that Solr throws an exception on any attempt to load a document
  48. # that contains a field that isn't specified in schema.xml.
  49. {
  50. sanitizeUnknownSolrFields {
  51. # Location from which to fetch Solr schema
  52. solrLocator : ${SOLR_LOCATOR}
  53. }
  54. }
  55. # log the record at DEBUG level to SLF4J
  56. { logDebug { format : "output record: {}", args : ["@{}"] } }
  57. # load the record into a Solr server or MapReduce Reducer
  58. {
  59. loadSolr {
  60. solrLocator : ${SOLR_LOCATOR}
  61. }
  62. }
  63. ]
  64. }
  65. ]