2( jupiter)

 def mapper(a, b):

    return [((i, j), a[i][k] * b[k][j])

            for i in range(len(a))

            for j in range(len(b[0]))

            for k in range(len(b))]

def map_reduce(a, b):

    kv = mapper(a, b)

    grouped = {}

    for k, v in kv:

        grouped.setdefault(k, []).append(v)

    return grouped

def to_matrix(grouped, r, c):

    return [[sum(grouped[(i, j)]) for j in range(c)] for i in range(r)]

# Example

a = [[1, 2, 3], [4, 5, 6]]

b = [[7, 8], [9, 10], [11, 12]]

for row in to_matrix(map_reduce(a, b), len(a), len(b[0])):

    print(row)


Comments

Post a Comment

Popular posts from this blog

7

 hive CREATE DATABASE employee_db; USE employee_db; CREATE TABLE employees (     emp_id INT,     name STRING,     age INT,     dept_id INT,     salary FLOAT ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE; LOAD DATA INPATH '/user/cloudera/employees.txt' INTO TABLE employees; ALTER TABLE employees ADD COLUMNS (email STRING); ALTER TABLE employees RENAME TO employees_new; DROP TABLE employees_new; CREATE VIEW high_salary_employees AS SELECT emp_id, name, salary FROM employees WHERE salary > 50000; ALTER VIEW high_salary_employees AS SELECT emp_id, name, age, salary FROM employees WHERE salary > 60000; DROP VIEW high_salary_employees; SHOW TABLES; DESCRIBE employees; SELECT * FROM employees LIMIT 5; -=
Read more

8

 nano wordcount_sim.py text = """ Hadoop MapReduce is a software framework for easily writing applications which process vast amounts """ def mapper(line):     words = line.strip().split()     return [(word, 1) for word in words] from collections import defaultdict # Map phase mapped = [] for line in text.strip().split('\n'):     mapped.extend(mapper(line)) # Shuffle and sort phase shuffle_sort = defaultdict(list) for word, count in mapped:     shuffle_sort[word].append(count) # Reduce phase def reducer(shuffled_data):     reduced = {}     for word, counts in shuffled_data.items():         reduced[word] = sum(counts)     return reduced word_counts = reducer(shuffle_sort) # Output for word, count in word_counts.items():     print(f"{word}\t{count}") ctrl+o python3 wordcount_sim.py (or)python wordcount_sim.py
Read more

6

 pig -x local students = LOAD '/home/cloudera/Desktop/students.txt' USING PigStorage(',')  AS (id:int, name:chararray, dept:chararray, marks:int); departments = LOAD '/home/cloudera/Desktop/departments.txt' USING PigStorage(',')  AS (dept_code:chararray, dept_name:chararray); high_scorers = FILTER students BY marks > 70; DUMP high_scorers; projected = FOREACH high_scorers GENERATE name, marks; DUMP projected; grouped_by_dept = GROUP students BY dept; DUMP grouped_by_dept; avg_marks = FOREACH grouped_by_dept GENERATE group AS department, AVG(students.marks) AS average_marks; DUMP avg_marks; joined = JOIN students BY dept, departments BY dept_code; DUMP joined; sorted = ORDER students BY marks DESC; DUMP sorted;
Read more