wordcount.py 2.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081
  1. #!/usr/bin/python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #
  18. # Wordcount example, for Hadoop streaming.
  19. #
  20. # Test with:
  21. # $(echo "hello moon"; echo "hello sun") | python wordcount.py map | sort | python wordcount.py reduce
  22. # hello 2
  23. # moon 1
  24. # sun 1
  25. import sys
  26. import re
  27. import __builtin__
  28. def map(line):
  29. for word in re.split("\W", line):
  30. if word:
  31. emit(word, str(1))
  32. def reduce(word, counts):
  33. emit(word, str(sum(__builtin__.map(int, counts))))
  34. def emit(key, value):
  35. """
  36. Emits a key->value pair. Key and value should be strings.
  37. """
  38. print "\t".join( (key, value) )
  39. def run_map():
  40. """Calls map() for each input value."""
  41. for line in sys.stdin:
  42. line = line.rstrip()
  43. map(line)
  44. def run_reduce():
  45. """Gathers reduce() data in memory, and calls reduce()."""
  46. prev_key = None
  47. values = []
  48. for line in sys.stdin:
  49. line = line.rstrip()
  50. key, value = re.split("\t", line, 1)
  51. if prev_key == key:
  52. values.append(value)
  53. else:
  54. if prev_key is not None:
  55. reduce(prev_key, values)
  56. prev_key = key
  57. values = [ value ]
  58. if prev_key is not None:
  59. reduce(prev_key, values)
  60. def main():
  61. """Runs map or reduce code, per arguments."""
  62. if len(sys.argv) != 2 or sys.argv[1] not in ("map", "reduce"):
  63. print "Usage: %s <map|reduce>" % sys.argv[0]
  64. sys.exit(1)
  65. if sys.argv[1] == "map":
  66. run_map()
  67. elif sys.argv[1] == "reduce":
  68. run_reduce()
  69. else:
  70. assert False
  71. if __name__ == "__main__":
  72. main()