streaming.py 2.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
  1. #!/usr/bin/python
  2. #
  3. # Licensed to Cloudera, Inc. under one
  4. # or more contributor license agreements. See the NOTICE file
  5. # distributed with this work for additional information
  6. # regarding copyright ownership. Cloudera, Inc. licenses this file
  7. # to you under the Apache License, Version 2.0 (the
  8. # "License"); you may not use this file except in compliance
  9. # with the License. You may obtain a copy of the License at
  10. #
  11. # http://www.apache.org/licenses/LICENSE-2.0
  12. #
  13. # Unless required by applicable law or agreed to in writing, software
  14. # distributed under the License is distributed on an "AS IS" BASIS,
  15. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16. # See the License for the specific language governing permissions and
  17. # limitations under the License.
  18. #
  19. #
  20. # Template for python Hadoop streaming. Fill in the map() and reduce()
  21. # functions, which should call emit(), as appropriate.
  22. #
  23. # Test your script with
  24. # cat input | python wordcount.py map | sort | python wordcount.py reduce
  25. import sys
  26. import re
  27. def map(line):
  28. # Fill this in!
  29. pass
  30. def reduce(key, values):
  31. # Fill this in
  32. pass
  33. # Common library code follows:
  34. def emit(key, value):
  35. """
  36. Emits a key->value pair. Key and value should be strings.
  37. """
  38. print "\t".join( (key, value) )
  39. def run_map():
  40. """Calls map() for each input value."""
  41. for line in sys.stdin:
  42. line = line.rstrip()
  43. map(line)
  44. def run_reduce():
  45. """Gathers reduce() data in memory, and calls reduce()."""
  46. prev_key = None
  47. values = []
  48. for line in sys.stdin:
  49. line = line.rstrip()
  50. key, value = re.split("\t", line, 1)
  51. if prev_key == key:
  52. values.append(value)
  53. else:
  54. if prev_key is not None:
  55. reduce(prev_key, values)
  56. prev_key = key
  57. values = [ value ]
  58. if prev_key is not None:
  59. reduce(prev_key, values)
  60. def main():
  61. """Runs map or reduce code, per arguments."""
  62. if len(sys.argv) != 2 or sys.argv[1] not in ("map", "reduce"):
  63. print "Usage: %s <map|reduce>" % sys.argv[0]
  64. sys.exit(1)
  65. if sys.argv[1] == "map":
  66. run_map()
  67. elif sys.argv[1] == "reduce":
  68. run_reduce()
  69. else:
  70. assert False
  71. if __name__ == "__main__":
  72. main()